diff --git a/Makefile.work b/Makefile.work index bece7e56cb74..d66774ddcf28 100644 --- a/Makefile.work +++ b/Makefile.work @@ -61,7 +61,9 @@ # * value to anything will enable it # * Default: unset # * Value: y -# +# * SONIC_PTF_ENV_PY_VER: Python version for PTF image +# * Default: mixed +# * Values: mixed,py3 ############################################################################### SHELL = /bin/bash @@ -592,6 +594,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ SONIC_OS_VERSION=$(SONIC_OS_VERSION) \ PIP_HTTP_TIMEOUT=$(PIP_HTTP_TIMEOUT) \ LEGACY_SONIC_MGMT_DOCKER=$(LEGACY_SONIC_MGMT_DOCKER) \ + SONIC_PTF_ENV_PY_VER=$(SONIC_PTF_ENV_PY_VER) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index 809b1474d6d1..d026b708b4b3 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -3,8 +3,10 @@ FROM {{ prefix }}multiarch/debian-debootstrap:armhf-stretch {% elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %} FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch +{% elif PTF_ENV_PY_VER == "mixed" %} +FROM {{ prefix}}debian:buster {% else %} -FROM {{ prefix }}debian:buster +FROM {{ prefix }}debian:bullseye {% endif %} {% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %} @@ -12,7 +14,7 @@ FROM {{ prefix }}debian:buster USER root WORKDIR /root -MAINTAINER Pavel Shirshov +LABEL maintainer="Pavel Shirshov" COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] @@ -45,18 +47,24 @@ RUN apt-get update \ cmake \ libqt5core5a \ libqt5network5 \ +{% if PTF_ENV_PY_VER == "mixed" %} libboost-atomic1.71.0 \ +{% else %} + libboost-atomic1.74.0 \ +{% endif %} less \ git \ iputils-ping \ hping3 \ curl \ tmux \ +{% if PTF_ENV_PY_VER == "mixed" %} python \ python-dev \ python-libpcap \ python-scapy \ python-six \ +{% endif %} python3 \ python3-venv \ python3-pip \ @@ -64,7 +72,10 @@ RUN apt-get update \ python3-scapy \ python3-six \ libpcap-dev \ +# TODO check if tacacs+ is required by tests +{% if PTF_ENV_PY_VER == "mixed" %} tacacs+ \ +{% endif %} rsyslog \ ntp \ ntpstat \ @@ -77,12 +88,20 @@ RUN apt-get update \ iproute2 \ wireshark-common +{% if PTF_ENV_PY_VER == "py3" %} +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ + && update-alternatives --install /usr/bin/pdb pdb /usr/bin/pdb3 1 \ + && update-alternatives --install /usr/bin/pydoc pydoc /usr/bin/pydoc3 1 \ + && update-alternatives --install /usr/bin/pygettext pygettext /usr/bin/pygettext3 1 +{% endif %} + # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy # TODO: Clean up this step RUN rm -rf /debs \ && apt-get -y autoclean \ && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* \ +{% if PTF_ENV_PY_VER == "mixed" %} && wget --https-only https://bootstrap.pypa.io/pip/2.7/get-pip.py \ && python get-pip.py \ && rm -f get-pip.py \ @@ -94,6 +113,7 @@ RUN rm -rf /debs \ && python setup.py install \ && cd .. \ && rm -fr scapy-vxlan \ +{% endif %} && git clone https://github.com/sflow/sflowtool \ && cd sflowtool \ && ./boot.sh \ @@ -121,6 +141,7 @@ RUN rm -rf /debs \ && cd ../.. \ && rm -fr nanomsg-1.0.0 \ && rm -f 1.0.0.tar.gz \ +{% if PTF_ENV_PY_VER == "mixed" %} && pip install cffi \ && pip install nnpy \ && pip install dpkt \ @@ -134,6 +155,7 @@ RUN rm -rf /debs \ && pip install unittest-xml-reporting \ && pip install pyrasite \ && pip install retrying \ +{% endif %} && mkdir -p /opt \ && cd /opt \ && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py @@ -147,10 +169,14 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8 +{% if PTF_ENV_PY_VER == "mixed" %} RUN python3 -m pip install --upgrade --ignore-installed pip +{% else %} +RUN pip install --upgrade --ignore-installed pip +{% endif %} # Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy -RUN python3 -m pip install setuptools \ +RUN pip3 install setuptools \ && pip3 install supervisor \ && pip3 install ipython==5.4.1 \ && pip3 install Cython \ @@ -200,9 +226,11 @@ COPY supervisord.conf /etc/supervisor/ COPY conf.d/ /etc/supervisor/conf.d/ COPY ptf_tgen.sh /ptf_tgen/ +{% if PTF_ENV_PY_VER == "mixed" %} # Move tcpdump into /usr/bin Otherwise it's impossible to run tcpdump due to a docker bug RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump +{% endif %} RUN mkdir -p /var/log/supervisor @@ -211,7 +239,12 @@ RUN git clone https://github.com/lguohan/gnxi.git \ && cd gnxi \ && git checkout 3adf8b9 \ && cd gnmi_cli_py \ +{% if PTF_ENV_PY_VER == "mixed" %} && pip install -r requirements.txt +{% else %} + && cat requirements.txt | grep -v futures > /tmp/requirements.txt \ + && pip3 install -r /tmp/requirements.txt +{% endif %} COPY \ {% for deb in docker_ptf_debs.split(' ') -%} diff --git a/platform/vs/docker-ptf.mk b/platform/vs/docker-ptf.mk index ef102be967f4..1c66f8b889bd 100644 --- a/platform/vs/docker-ptf.mk +++ b/platform/vs/docker-ptf.mk @@ -3,6 +3,10 @@ DOCKER_PTF = docker-ptf.gz $(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3) $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf +ifeq ($(SONIC_PTF_ENV_PY_VER), mixed) $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT) +else +$(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT_0_14_1) $(PYTHON3_THRIFT_0_14_1) $(PYTHON_SAITHRIFT) +endif SONIC_DOCKER_IMAGES += $(DOCKER_PTF) SONIC_BUSTER_DOCKERS += $(DOCKER_PTF) diff --git a/rules/config b/rules/config index e52eeae4c410..2996d0f46f35 100644 --- a/rules/config +++ b/rules/config @@ -319,3 +319,7 @@ PIP_HTTP_TIMEOUT ?= 60 # BUILD_REDUCE_IMAGE_SIZE - reduce the image size as much as possbible BUILD_REDUCE_IMAGE_SIZE = n + +# SONIC_PTF_ENV_PY_VER - SONiC PTF test Python version. Set to 'mixed' to build the +# image with both Python 2 and 3. Set to 'py3' to build a Python 3 only image +SONIC_PTF_ENV_PY_VER = mixed diff --git a/slave.mk b/slave.mk index eb537e740527..c6c0e32e6569 100644 --- a/slave.mk +++ b/slave.mk @@ -75,6 +75,9 @@ else ENABLE_PY2_MODULES = y endif +# Python version for PTF image +PTF_ENV_PY_VER = $(if $(SONIC_PTF_ENV_PY_VER),$(SONIC_PTF_ENV_PY_VER),mixed) + export BUILD_NUMBER export BUILD_TIMESTAMP export SONIC_IMAGE_VERSION @@ -94,6 +97,7 @@ export MIRROR_SNAPSHOT export SONIC_OS_VERSION export FILES_PATH export PROJECT_ROOT +export PTF_ENV_PY_VER ############################################################################### ## Utility rules @@ -461,6 +465,7 @@ $(info "CROSS_BUILD_ENVIRON" : "$(CROSS_BUILD_ENVIRON)") $(info "GZ_COMPRESS_PROGRAM" : "$(GZ_COMPRESS_PROGRAM)") $(info "LEGACY_SONIC_MGMT_DOCKER" : "$(LEGACY_SONIC_MGMT_DOCKER)") $(info "INCLUDE_EXTERNAL_PATCHES" : "$(INCLUDE_EXTERNAL_PATCHES)") +$(info "PTF_ENV_PY_VER" : "$(PTF_ENV_PY_VER)") $(info ) else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))