diff --git a/.gitmodules b/.gitmodules index 9194c8f4b7a2..59cddea4a8f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -103,3 +103,6 @@ [submodule "src/sonic-p4rt/sonic-pins"] path = src/sonic-p4rt/sonic-pins url = https://github.com/Azure/sonic-pins.git +[submodule "src/ptf-py3"] + path = src/ptf-py3 + url = https://github.com/p4lang/ptf.git diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index d999d01e0026..bf22ba97ea06 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -7,6 +7,11 @@ FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch FROM {{ prefix }}debian:buster {% endif %} +{% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %} + +USER root +WORKDIR /root + MAINTAINER Pavel Shirshov RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list @@ -51,6 +56,13 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' / python-libpcap \ python-scapy \ python-six \ + python3 \ + python3-venv \ + python3-pip \ + python3-dev \ + python3-scapy \ + python3-six \ + libpcap-dev \ tacacs+ \ rsyslog \ ntp \ @@ -59,7 +71,9 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' / arping \ bridge-utils \ libteam-utils \ - gdb + gdb \ + automake \ + iproute2 # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy # TODO: Clean up this step @@ -117,10 +131,61 @@ RUN rm -rf /debs \ && pip install pybrctl pyro4 rpyc yabgp \ && pip install unittest-xml-reporting \ && pip install pyrasite \ + && pip install retrying \ && mkdir -p /opt \ && cd /opt \ && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py +RUN python3 -m venv env-python3 + +# Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD. +ENV VIRTUAL_ENV=/root/env-python3 +ARG BACKUP_OF_PATH="$PATH" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8 + +RUN python3 -m pip install --upgrade --ignore-installed pip + +# Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy +RUN python3 -m pip install setuptools \ + && pip3 install supervisor \ + && pip3 install ipython==5.4.1 \ + && pip3 install Cython \ + && pip3 install cffi \ + && pip3 install nnpy \ + && pip3 install dpkt \ + && pip3 install ipaddress \ + && pip3 install pysubnettree \ + && pip3 install paramiko \ + && pip3 install Flask \ + && pip3 install exabgp \ + && pip3 install pyaml \ + && pip3 install pybrctl pyro4 rpyc yabgp \ + && pip3 install unittest-xml-reporting \ + && pip3 install pyrasite \ + && pip3 install python-libpcap \ + && pip3 install enum34 \ + && pip3 install grpcio \ + && pip3 install grpcio-tools \ + && pip3 install protobuf \ + && pip3 install six==1.16.0 \ + && pip3 install itsdangerous \ + && pip3 install retrying \ + && pip3 install jinja2 \ + && pip3 install scapy==2.4.5 + +{% if docker_ptf_whls.strip() -%} +# Copy locally-built Python wheel dependencies +{{ copy_files("python-wheels/", docker_ptf_whls.split(' '), "/python-wheels/") }} + +# Install locally-built Python wheel dependencies +{{ install_python_wheels(docker_ptf_whls.split(' ')) }} +{% endif %} + +# Deactivating a virtualenv. +ENV PATH="$BACKUP_OF_PATH" + ## Adjust sshd settings RUN mkdir /var/run/sshd \ && echo 'root:root' | chpasswd \ diff --git a/platform/vs/docker-ptf.mk b/platform/vs/docker-ptf.mk index 445fc8daf4d3..ef102be967f4 100644 --- a/platform/vs/docker-ptf.mk +++ b/platform/vs/docker-ptf.mk @@ -1,6 +1,7 @@ # docker image for docker-ptf DOCKER_PTF = docker-ptf.gz +$(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3) $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT) SONIC_DOCKER_IMAGES += $(DOCKER_PTF) diff --git a/rules/ptf-py3.dep b/rules/ptf-py3.dep new file mode 100644 index 000000000000..eadb32aabdf4 --- /dev/null +++ b/rules/ptf-py3.dep @@ -0,0 +1,11 @@ + +SPATH := $($(PTF_PY3)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/ptf-py3.mk rules/ptf-py3.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files)) + +$(PTF_PY3)_CACHE_MODE := GIT_CONTENT_SHA +$(PTF_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(PTF_PY3)_DEP_FILES := $(DEP_FILES) +$(PTF_PY3)_SMDEP_FILES := $(SMDEP_FILES) +$(PTF_PY3)_SMDEP_PATHS := $(SPATH) diff --git a/rules/ptf-py3.mk b/rules/ptf-py3.mk new file mode 100644 index 000000000000..bf583b57b622 --- /dev/null +++ b/rules/ptf-py3.mk @@ -0,0 +1,7 @@ +# ptf package + +PTF_PY3 = ptf-0.9.3-py3-none-any.whl +$(PTF_PY3)_SRC_PATH = $(SRC_PATH)/ptf-py3 +$(PTF_PY3)_PYTHON_VERSION = 3 +$(PTF_PY3)_TEST = n +SONIC_PYTHON_WHEELS += $(PTF_PY3) diff --git a/src/ptf-py3 b/src/ptf-py3 new file mode 160000 index 000000000000..405513bcad2e --- /dev/null +++ b/src/ptf-py3 @@ -0,0 +1 @@ +Subproject commit 405513bcad2eae3092b0ac4ceb31e8dec5e32311 diff --git a/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch b/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch new file mode 100644 index 000000000000..cb2b992548a7 --- /dev/null +++ b/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch @@ -0,0 +1,26 @@ +From a8b13b9fbaa16ddd305ba2df2238ef606ef222a7 Mon Sep 17 00:00:00 2001 +From: Zhaohui Sun +Date: Wed, 13 Apr 2022 09:24:46 +0000 +Subject: [PATCH 1/2] Remove ord in get_mac() to avoid TypeError + +Signed-off-by: Zhaohui Sun +--- + src/ptf/netutils.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/ptf/netutils.py b/src/ptf/netutils.py +index 752e76c..6aabe79 100644 +--- a/src/ptf/netutils.py ++++ b/src/ptf/netutils.py +@@ -54,7 +54,7 @@ def get_if_index(iff): + + + def get_mac(iff): +- return ":".join(["%02x" % ord(char) for char in get_if(iff, SIOCGIFHWADDR)[18:24]]) ++ return ":".join(["%02x" % char for char in get_if(iff, SIOCGIFHWADDR)[18:24]]) + + + def set_promisc(s, iff, val=1): +-- +2.25.1 + diff --git a/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch b/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch new file mode 100644 index 000000000000..a0582994539e --- /dev/null +++ b/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch @@ -0,0 +1,36 @@ +From 6e570e00ea05882d2db1e480ed041ea631bf37da Mon Sep 17 00:00:00 2001 +From: Zhaohui Sun +Date: Wed, 13 Apr 2022 09:25:28 +0000 +Subject: [PATCH 2/2] Fill byte formatted client mac address in DHCP Discover + packet + +Signed-off-by: Zhaohui Sun +--- + src/ptf/testutils.py | 9 +++++---- + 1 file changed, 5 insertions(+), 4 deletions(-) + +diff --git a/src/ptf/testutils.py b/src/ptf/testutils.py +index ab67cea..83a9075 100755 +--- a/src/ptf/testutils.py ++++ b/src/ptf/testutils.py +@@ -2712,12 +2712,13 @@ def __dhcp_mac_to_chaddr(mac_addr="00:01:02:03:04:05"): + """ + Private helper function to convert a 6-byte MAC address of form: + '00:01:02:03:04:05' +- into a 16-byte chaddr byte string of form: +- '\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' ++ into a 16-byte chaddr byte of form: ++ b'\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + + """ +- chaddr = "".join([chr(int(octet, 16)) for octet in mac_addr.split(":")]) +- chaddr += "\x00" * 10 ++ import binascii ++ chaddr = binascii.unhexlify(mac_addr.replace(':', '')) ++ chaddr += b'\x00\x00\x00\x00\x00\x00' + return chaddr + + +-- +2.25.1 + diff --git a/src/ptf-py3.patch/series b/src/ptf-py3.patch/series new file mode 100644 index 000000000000..d18987dd5913 --- /dev/null +++ b/src/ptf-py3.patch/series @@ -0,0 +1,2 @@ +0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch +0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch \ No newline at end of file