Add python3 virtual environment for docker-ptf (#10599)

Why I did it
Migrate ptftests script to python3, in order to do an incremental migration, add python virtual environment firstly, install all required python packages in virtual env as well.
Then migrate ptftests scripts from python2 to python3 one by one avoid impacting non-changed scripts.

Signed-off-by: Zhaohui Sun zhaohuisun@microsoft.com

How I did it
Add python3 virtual environment for docker-ptf.
Add submodule ptf-py3 and install patched ptf 0.9.3 into virtual environment as well, two ptf issues were reported here:
p4lang/ptf#173
p4lang/ptf#174

Signed-off-by: Zhaohui Sun <zhaohuisun@microsoft.com>

.gitmodules

@@ -103,3 +103,6 @@
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
 	url = https://github.com/Azure/sonic-pins.git
+[submodule "src/ptf-py3"]
+	path = src/ptf-py3
+	url = https://github.com/p4lang/ptf.git

dockers/docker-ptf/Dockerfile.j2

@@ -7,6 +7,11 @@ FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch
 FROM {{ prefix }}debian:buster
 {% endif %}
 
+{% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %}
+
+USER root
+WORKDIR /root
+
 MAINTAINER Pavel Shirshov
 
 RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list
@@ -51,6 +56,13 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /
         python-libpcap      \
         python-scapy        \
         python-six          \
+        python3             \
+        python3-venv        \
+        python3-pip         \
+        python3-dev         \
+        python3-scapy       \
+        python3-six         \
+        libpcap-dev         \
         tacacs+             \
         rsyslog             \
         ntp                 \
@@ -59,7 +71,9 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /
         arping              \
         bridge-utils        \
         libteam-utils       \
-        gdb
+        gdb                 \
+        automake            \
+        iproute2
 
 # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy
 # TODO: Clean up this step
@@ -117,10 +131,61 @@ RUN rm -rf /debs \
     && pip install pybrctl pyro4 rpyc yabgp \
     && pip install unittest-xml-reporting \
     && pip install pyrasite \
+    && pip install retrying \
     && mkdir -p /opt       \
     && cd /opt             \
     && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py
 
+RUN python3 -m venv env-python3
+
+# Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD.
+ENV VIRTUAL_ENV=/root/env-python3
+ARG BACKUP_OF_PATH="$PATH"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8
+
+RUN python3 -m pip install --upgrade --ignore-installed pip
+
+# Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy
+RUN python3 -m pip install setuptools \
+    && pip3 install supervisor \
+    && pip3 install ipython==5.4.1 \
+    && pip3 install Cython \
+    && pip3 install cffi    \
+    && pip3 install nnpy    \
+    && pip3 install dpkt    \
+    && pip3 install ipaddress \
+    && pip3 install pysubnettree \
+    && pip3 install paramiko \
+    && pip3 install Flask   \
+    && pip3 install exabgp \
+    && pip3 install pyaml   \
+    && pip3 install pybrctl pyro4 rpyc yabgp \
+    && pip3 install unittest-xml-reporting \
+    && pip3 install pyrasite \
+    && pip3 install python-libpcap \
+    && pip3 install enum34 \
+    && pip3 install grpcio \
+    && pip3 install grpcio-tools \
+    && pip3 install protobuf \
+    && pip3 install six==1.16.0 \
+    && pip3 install itsdangerous \
+    && pip3 install retrying \
+    && pip3 install jinja2 \
+    && pip3 install scapy==2.4.5
+
+{% if docker_ptf_whls.strip() -%}
+# Copy locally-built Python wheel dependencies
+{{ copy_files("python-wheels/", docker_ptf_whls.split(' '), "/python-wheels/") }}
+
+# Install locally-built Python wheel dependencies
+{{ install_python_wheels(docker_ptf_whls.split(' ')) }}
+{% endif %}
+
+# Deactivating a virtualenv.
+ENV PATH="$BACKUP_OF_PATH"
+
 ## Adjust sshd settings
 RUN mkdir /var/run/sshd \
     && echo 'root:root' | chpasswd \

platform/vs/docker-ptf.mk

@@ -1,6 +1,7 @@
 # docker image for docker-ptf
 
 DOCKER_PTF = docker-ptf.gz
+$(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3)
 $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf
 $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT)
 SONIC_DOCKER_IMAGES += $(DOCKER_PTF)

rules/ptf-py3.dep

@@ -0,0 +1,11 @@
+
+SPATH       := $($(PTF_PY3)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/ptf-py3.mk rules/ptf-py3.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(PTF_PY3)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(PTF_PY3)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(PTF_PY3)_DEP_FILES   := $(DEP_FILES)
+$(PTF_PY3)_SMDEP_FILES := $(SMDEP_FILES)
+$(PTF_PY3)_SMDEP_PATHS := $(SPATH)

rules/ptf-py3.mk

@@ -0,0 +1,7 @@
+# ptf package
+
+PTF_PY3 = ptf-0.9.3-py3-none-any.whl
+$(PTF_PY3)_SRC_PATH = $(SRC_PATH)/ptf-py3
+$(PTF_PY3)_PYTHON_VERSION = 3
+$(PTF_PY3)_TEST = n
+SONIC_PYTHON_WHEELS += $(PTF_PY3)

src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch

@@ -0,0 +1,26 @@
+From a8b13b9fbaa16ddd305ba2df2238ef606ef222a7 Mon Sep 17 00:00:00 2001
+From: Zhaohui Sun <zhaohuisun@microsoft.com>
+Date: Wed, 13 Apr 2022 09:24:46 +0000
+Subject: [PATCH 1/2] Remove ord in get_mac() to avoid TypeError
+
+Signed-off-by: Zhaohui Sun <zhaohuisun@microsoft.com>
+---
+ src/ptf/netutils.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/ptf/netutils.py b/src/ptf/netutils.py
+index 752e76c..6aabe79 100644
+--- a/src/ptf/netutils.py
++++ b/src/ptf/netutils.py
+@@ -54,7 +54,7 @@ def get_if_index(iff):
+
+
+ def get_mac(iff):
+-    return ":".join(["%02x" % ord(char) for char in get_if(iff, SIOCGIFHWADDR)[18:24]])
++    return ":".join(["%02x" % char for char in get_if(iff, SIOCGIFHWADDR)[18:24]])
+
+
+ def set_promisc(s, iff, val=1):
+-- 
+2.25.1
+

src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch

@@ -0,0 +1,36 @@
+From 6e570e00ea05882d2db1e480ed041ea631bf37da Mon Sep 17 00:00:00 2001
+From: Zhaohui Sun <zhaohuisun@microsoft.com>
+Date: Wed, 13 Apr 2022 09:25:28 +0000
+Subject: [PATCH 2/2] Fill byte formatted client mac address in DHCP Discover
+ packet
+
+Signed-off-by: Zhaohui Sun <zhaohuisun@microsoft.com>
+---
+ src/ptf/testutils.py | 9 +++++----
+ 1 file changed, 5 insertions(+), 4 deletions(-)
+
+diff --git a/src/ptf/testutils.py b/src/ptf/testutils.py
+index ab67cea..83a9075 100755
+--- a/src/ptf/testutils.py
++++ b/src/ptf/testutils.py
+@@ -2712,12 +2712,13 @@ def __dhcp_mac_to_chaddr(mac_addr="00:01:02:03:04:05"):
+     """
+     Private helper function to convert a 6-byte MAC address of form:
+       '00:01:02:03:04:05'
+-    into a 16-byte chaddr byte string of form:
+-      '\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
++    into a 16-byte chaddr byte of form:
++      b'\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+
+     """
+-    chaddr = "".join([chr(int(octet, 16)) for octet in mac_addr.split(":")])
+-    chaddr += "\x00" * 10
++    import binascii
++    chaddr = binascii.unhexlify(mac_addr.replace(':', ''))
++    chaddr += b'\x00\x00\x00\x00\x00\x00'
+     return chaddr
+
+
+-- 
+2.25.1
+

src/ptf-py3.patch/series

@@ -0,0 +1,2 @@
+0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch
+0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch