From 2b8e15ef0cf247372355b5fc5678ffd727571fd5 Mon Sep 17 00:00:00 2001 From: Zhaohui Sun Date: Mon, 18 Apr 2022 08:16:02 +0000 Subject: [PATCH 1/6] Add python3 virtual environment for docker-ptf Signed-off-by: Zhaohui Sun --- .gitmodules | 3 + dockers/docker-ptf/Dockerfile.j2 | 65 ++++++++++++++++++- platform/vs/docker-ptf.mk | 1 + rules/ptf-py3.dep | 11 ++++ rules/ptf-py3.mk | 7 ++ src/ptf-py3 | 1 + ...ve-ord-in-get_mac-to-avoid-TypeError.patch | 26 ++++++++ ...ted-client-mac-address-in-DHCP-Disco.patch | 36 ++++++++++ src/ptf-py3.patch/series | 2 + 9 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 rules/ptf-py3.dep create mode 100644 rules/ptf-py3.mk create mode 160000 src/ptf-py3 create mode 100644 src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch create mode 100644 src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch create mode 100644 src/ptf-py3.patch/series diff --git a/.gitmodules b/.gitmodules index 9194c8f4b7a2..59cddea4a8f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -103,3 +103,6 @@ [submodule "src/sonic-p4rt/sonic-pins"] path = src/sonic-p4rt/sonic-pins url = https://github.com/Azure/sonic-pins.git +[submodule "src/ptf-py3"] + path = src/ptf-py3 + url = https://github.com/p4lang/ptf.git diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index d999d01e0026..019200458118 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -7,6 +7,8 @@ FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch FROM {{ prefix }}debian:buster {% endif %} +{% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %} + MAINTAINER Pavel Shirshov RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list @@ -51,6 +53,13 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' / python-libpcap \ python-scapy \ python-six \ + python3 \ + python3-venv \ + python3-pip \ + python3-dev \ + python3-scapy \ + python3-six \ + libpcap-dev \ tacacs+ \ rsyslog \ ntp \ @@ -59,7 +68,9 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' / arping \ bridge-utils \ libteam-utils \ - gdb + gdb \ + automake \ + iproute2 # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy # TODO: Clean up this step @@ -117,10 +128,62 @@ RUN rm -rf /debs \ && pip install pybrctl pyro4 rpyc yabgp \ && pip install unittest-xml-reporting \ && pip install pyrasite \ + && pip install retrying \ && mkdir -p /opt \ && cd /opt \ && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py +RUN python3 -m venv env-python3 + +# Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD. +ENV VIRTUAL_ENV=env-python3 +ARG BACKUP_OF_PATH="$PATH" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8 + +RUN python3 -m pip install --upgrade --ignore-installed pip + +# Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy +# TODO: Clean up this step +RUN python3 -m pip install setuptools \ + && pip3 install supervisor \ + && pip3 install ipython==5.4.1 \ + && pip3 install Cython \ + && pip3 install cffi \ + && pip3 install nnpy \ + && pip3 install dpkt \ + && pip3 install ipaddress \ + && pip3 install pysubnettree \ + && pip3 install paramiko \ + && pip3 install Flask \ + && pip3 install exabgp \ + && pip3 install pyaml \ + && pip3 install pybrctl pyro4 rpyc yabgp \ + && pip3 install unittest-xml-reporting \ + && pip3 install pyrasite \ + && pip3 install python-libpcap \ + && pip3 install enum34 \ + && pip3 install grpcio \ + && pip3 install grpcio-tools \ + && pip3 install protobuf \ + && pip3 install six \ + && pip3 install itsdangerous \ + && pip3 install retrying \ + && pip3 install jinja2 \ + && pip3 install scapy==2.4.5 + +{% if docker_ptf_whls.strip() -%} +# Copy locally-built Python wheel dependencies +{{ copy_files("python-wheels/", docker_ptf_whls.split(' '), "/python-wheels/") }} + +# Install locally-built Python wheel dependencies +{{ install_python_wheels(docker_ptf_whls.split(' ')) }} +{% endif %} + +# Deactivating a virtualenv. +ENV PATH="$BACKUP_OF_PATH" + ## Adjust sshd settings RUN mkdir /var/run/sshd \ && echo 'root:root' | chpasswd \ diff --git a/platform/vs/docker-ptf.mk b/platform/vs/docker-ptf.mk index 445fc8daf4d3..ef102be967f4 100644 --- a/platform/vs/docker-ptf.mk +++ b/platform/vs/docker-ptf.mk @@ -1,6 +1,7 @@ # docker image for docker-ptf DOCKER_PTF = docker-ptf.gz +$(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3) $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT) SONIC_DOCKER_IMAGES += $(DOCKER_PTF) diff --git a/rules/ptf-py3.dep b/rules/ptf-py3.dep new file mode 100644 index 000000000000..eadb32aabdf4 --- /dev/null +++ b/rules/ptf-py3.dep @@ -0,0 +1,11 @@ + +SPATH := $($(PTF_PY3)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/ptf-py3.mk rules/ptf-py3.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files)) + +$(PTF_PY3)_CACHE_MODE := GIT_CONTENT_SHA +$(PTF_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(PTF_PY3)_DEP_FILES := $(DEP_FILES) +$(PTF_PY3)_SMDEP_FILES := $(SMDEP_FILES) +$(PTF_PY3)_SMDEP_PATHS := $(SPATH) diff --git a/rules/ptf-py3.mk b/rules/ptf-py3.mk new file mode 100644 index 000000000000..bf583b57b622 --- /dev/null +++ b/rules/ptf-py3.mk @@ -0,0 +1,7 @@ +# ptf package + +PTF_PY3 = ptf-0.9.3-py3-none-any.whl +$(PTF_PY3)_SRC_PATH = $(SRC_PATH)/ptf-py3 +$(PTF_PY3)_PYTHON_VERSION = 3 +$(PTF_PY3)_TEST = n +SONIC_PYTHON_WHEELS += $(PTF_PY3) diff --git a/src/ptf-py3 b/src/ptf-py3 new file mode 160000 index 000000000000..405513bcad2e --- /dev/null +++ b/src/ptf-py3 @@ -0,0 +1 @@ +Subproject commit 405513bcad2eae3092b0ac4ceb31e8dec5e32311 diff --git a/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch b/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch new file mode 100644 index 000000000000..cb2b992548a7 --- /dev/null +++ b/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch @@ -0,0 +1,26 @@ +From a8b13b9fbaa16ddd305ba2df2238ef606ef222a7 Mon Sep 17 00:00:00 2001 +From: Zhaohui Sun +Date: Wed, 13 Apr 2022 09:24:46 +0000 +Subject: [PATCH 1/2] Remove ord in get_mac() to avoid TypeError + +Signed-off-by: Zhaohui Sun +--- + src/ptf/netutils.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/ptf/netutils.py b/src/ptf/netutils.py +index 752e76c..6aabe79 100644 +--- a/src/ptf/netutils.py ++++ b/src/ptf/netutils.py +@@ -54,7 +54,7 @@ def get_if_index(iff): + + + def get_mac(iff): +- return ":".join(["%02x" % ord(char) for char in get_if(iff, SIOCGIFHWADDR)[18:24]]) ++ return ":".join(["%02x" % char for char in get_if(iff, SIOCGIFHWADDR)[18:24]]) + + + def set_promisc(s, iff, val=1): +-- +2.25.1 + diff --git a/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch b/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch new file mode 100644 index 000000000000..a0582994539e --- /dev/null +++ b/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch @@ -0,0 +1,36 @@ +From 6e570e00ea05882d2db1e480ed041ea631bf37da Mon Sep 17 00:00:00 2001 +From: Zhaohui Sun +Date: Wed, 13 Apr 2022 09:25:28 +0000 +Subject: [PATCH 2/2] Fill byte formatted client mac address in DHCP Discover + packet + +Signed-off-by: Zhaohui Sun +--- + src/ptf/testutils.py | 9 +++++---- + 1 file changed, 5 insertions(+), 4 deletions(-) + +diff --git a/src/ptf/testutils.py b/src/ptf/testutils.py +index ab67cea..83a9075 100755 +--- a/src/ptf/testutils.py ++++ b/src/ptf/testutils.py +@@ -2712,12 +2712,13 @@ def __dhcp_mac_to_chaddr(mac_addr="00:01:02:03:04:05"): + """ + Private helper function to convert a 6-byte MAC address of form: + '00:01:02:03:04:05' +- into a 16-byte chaddr byte string of form: +- '\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' ++ into a 16-byte chaddr byte of form: ++ b'\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + + """ +- chaddr = "".join([chr(int(octet, 16)) for octet in mac_addr.split(":")]) +- chaddr += "\x00" * 10 ++ import binascii ++ chaddr = binascii.unhexlify(mac_addr.replace(':', '')) ++ chaddr += b'\x00\x00\x00\x00\x00\x00' + return chaddr + + +-- +2.25.1 + diff --git a/src/ptf-py3.patch/series b/src/ptf-py3.patch/series new file mode 100644 index 000000000000..d18987dd5913 --- /dev/null +++ b/src/ptf-py3.patch/series @@ -0,0 +1,2 @@ +0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch +0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch \ No newline at end of file From b92bb186c39a5d8a8b99c7004a65f89d085fede5 Mon Sep 17 00:00:00 2001 From: Zhaohui Sun Date: Tue, 19 Apr 2022 07:41:28 +0000 Subject: [PATCH 2/6] Use absolute path for virtual python3 env Signed-off-by: Zhaohui Sun --- dockers/docker-ptf/Dockerfile.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index 019200458118..a5e25f055420 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -9,6 +9,9 @@ FROM {{ prefix }}debian:buster {% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %} +USER root +WORKDIR /root + MAINTAINER Pavel Shirshov RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list @@ -136,7 +139,7 @@ RUN rm -rf /debs \ RUN python3 -m venv env-python3 # Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD. -ENV VIRTUAL_ENV=env-python3 +ENV VIRTUAL_ENV=/root/env-python3 ARG BACKUP_OF_PATH="$PATH" ENV PATH="$VIRTUAL_ENV/bin:$PATH" From d9069faf135fe80f6a5ddb1e03604f9d2864bd01 Mon Sep 17 00:00:00 2001 From: Zhaohui Sun Date: Wed, 20 Apr 2022 00:59:02 +0000 Subject: [PATCH 3/6] add version number for six Signed-off-by: Zhaohui Sun --- dockers/docker-ptf/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index a5e25f055420..dbb860d225df 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -170,7 +170,7 @@ RUN python3 -m pip install setuptools \ && pip3 install grpcio \ && pip3 install grpcio-tools \ && pip3 install protobuf \ - && pip3 install six \ + && pip3 install six==1.16.0 \ && pip3 install itsdangerous \ && pip3 install retrying \ && pip3 install jinja2 \ From 374ff57098e3d1059ecd450beb62572e173d1335 Mon Sep 17 00:00:00 2001 From: Zhaohui Sun Date: Thu, 21 Apr 2022 06:06:39 +0000 Subject: [PATCH 4/6] Update comments Signed-off-by: Zhaohui Sun --- dockers/docker-ptf/Dockerfile.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index dbb860d225df..bf22ba97ea06 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -147,8 +147,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8 RUN python3 -m pip install --upgrade --ignore-installed pip -# Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy -# TODO: Clean up this step +# Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy RUN python3 -m pip install setuptools \ && pip3 install supervisor \ && pip3 install ipython==5.4.1 \ From edbd217f81bd53eb6a6ff9fe8fdf3ce20bd2f6bc Mon Sep 17 00:00:00 2001 From: Zhaohui Sun Date: Fri, 22 Apr 2022 05:12:00 +0000 Subject: [PATCH 5/6] add some packages which installed in py2 environment Signed-off-by: Zhaohui Sun --- dockers/docker-ptf/Dockerfile.j2 | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index bf22ba97ea06..ab53a3bf2ea2 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -152,6 +152,33 @@ RUN python3 -m pip install setuptools \ && pip3 install supervisor \ && pip3 install ipython==5.4.1 \ && pip3 install Cython \ + && git clone https://github.com/sflow/sflowtool \ + && cd sflowtool \ + && ./boot.sh \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr sflowtool \ + && git clone https://github.com/dyninc/OpenBFDD.git \ + && cd OpenBFDD \ + && ./autogen.sh \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr OpenBFDD \ + && wget https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz \ + && tar xvfz 1.0.0.tar.gz \ + && cd nanomsg-1.0.0 \ + && mkdir -p build \ + && cd build \ + && cmake .. \ + && make install \ + && ldconfig \ + && cd ../.. \ + && rm -fr nanomsg-1.0.0 \ + && rm -f 1.0.0.tar.gz \ && pip3 install cffi \ && pip3 install nnpy \ && pip3 install dpkt \ From 8dfd8500f8a33155f4aff9748bc60a7fd2892d3d Mon Sep 17 00:00:00 2001 From: Zhaohui Sun Date: Mon, 25 Apr 2022 07:24:40 +0000 Subject: [PATCH 6/6] Remove sflowtool, openBFDD and nanomsg Signed-off-by: Zhaohui Sun --- dockers/docker-ptf/Dockerfile.j2 | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index ab53a3bf2ea2..bf22ba97ea06 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -152,33 +152,6 @@ RUN python3 -m pip install setuptools \ && pip3 install supervisor \ && pip3 install ipython==5.4.1 \ && pip3 install Cython \ - && git clone https://github.com/sflow/sflowtool \ - && cd sflowtool \ - && ./boot.sh \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -fr sflowtool \ - && git clone https://github.com/dyninc/OpenBFDD.git \ - && cd OpenBFDD \ - && ./autogen.sh \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -fr OpenBFDD \ - && wget https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz \ - && tar xvfz 1.0.0.tar.gz \ - && cd nanomsg-1.0.0 \ - && mkdir -p build \ - && cd build \ - && cmake .. \ - && make install \ - && ldconfig \ - && cd ../.. \ - && rm -fr nanomsg-1.0.0 \ - && rm -f 1.0.0.tar.gz \ && pip3 install cffi \ && pip3 install nnpy \ && pip3 install dpkt \