HITL test fixups (commaai#1117)

* automated -> hitl

* simplify

* jenkinsfile cleanup

* cleanup

* fix new linter errors

* fix ws cleanup

* some retry

* more cleaning up after ourselves

* unpin scons

* bump opendbc

* fix quotes

* enable flaky

* debug print

Co-authored-by: Bruce Wayne <batman@comma.ai>

.github/workflows/test.yaml

@@ -10,7 +10,7 @@ env:
   PERSIST: docker run -v $GITHUB_WORKSPACE:/tmp/openpilot/panda -w /tmp/openpilot/panda --name panda panda /bin/bash -c
   BUILD: |
       export DOCKER_BUILDKIT=1
-      docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from ghcr.io/commaai/panda:latest -t panda -f Dockerfile.panda .
+      docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from ghcr.io/commaai/panda:latest -t panda -f Dockerfile .
 
 jobs:
   docker_push:

.pre-commit-config.yaml

@@ -10,14 +10,12 @@ repos:
     rev: v0.910-1
     hooks:
     -   id: mypy
-        exclude: '^(tests/automated)/'
         additional_dependencies: ['git+https://github.com/numpy/numpy-stubs', 'types-requests', 'types-atomicwrites',
                                   'types-pycurl']
 -   repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:
     -   id: flake8
-        exclude: '^(tests/automated)/'
         args:
         - --select=F,E112,E113,E304,E501,E502,E701,E702,E703,E71,E72,E731,W191,W6
         - --exclude=tests/gmbitbang/*
@@ -30,7 +28,9 @@ repos:
         entry: pylint
         language: system
         types: [python]
-        exclude: '^(tests/automated)/'
         args:
+        - -rn
+        - -sn
+        - -j0
         - --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136
         - --generated-members="usb1.*"

Dockerfile

@@ -1,73 +1,91 @@
 FROM ubuntu:20.04
 ENV PYTHONUNBUFFERED 1
+ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     autoconf \
     automake \
-    bash \
-    bison \
     bzip2 \
     ca-certificates \
+    capnproto \
+    clang \
     curl \
-    dfu-util \
-    flex \
     g++ \
-    gawk \
-    gcc \
+    gcc-arm-none-eabi libnewlib-arm-none-eabi \
     git \
-    gperf \
-    help2man \
-    iputils-ping \
+    libarchive-dev \
+    libavformat-dev libavcodec-dev libavdevice-dev libavutil-dev libswscale-dev libavresample-dev libavfilter-dev \
     libbz2-dev \
-    libexpat-dev \
+    libcapnp-dev \
+    libcurl4-openssl-dev \
     libffi-dev \
-    libssl-dev \
-    libstdc++-arm-none-eabi-newlib \
     libtool \
-    libtool-bin \
+    libssl-dev \
+    libsqlite3-dev \
     libusb-1.0-0 \
-    locales  \
+    libzmq3-dev \
+    locales \
+    opencl-headers \
+    ocl-icd-opencl-dev \
     make \
-    ncurses-dev \
-    network-manager \
+    patch \
+    pkg-config \
+    python \
     python-dev \
-    python3-serial \
-    sed \
-    texinfo \
-    unrar-free \
     unzip \
     wget \
-    build-essential \
-    python-dev \
-    screen \
-    vim \
-    wget \
-    wireless-tools \
-    zlib1g-dev
+    zlib1g-dev \
+ && rm -rf /var/lib/apt/lists/* && \
+    cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \
+    rm -rf arm/ && \
+    rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-RUN pyenv install 3.8.10
-RUN pyenv global 3.8.10
-RUN pyenv rehash
 
-RUN pip install --upgrade pip==18.0
+ENV PANDA_PATH=/tmp/openpilot/panda
+ENV OPENPILOT_REF="ee0dd36a3c775dbd82493c84f4e7272c1eb3fcbd"
+ENV OPENDBC_REF="296f190000a2e71408e207ba21a2257cc853ec15"
 
 COPY requirements.txt /tmp/
-RUN pip install -r /tmp/requirements.txt
+RUN pyenv install 3.8.10 && \
+    pyenv global 3.8.10 && \
+    pyenv rehash && \
+    pip install --no-cache-dir -r /tmp/requirements.txt
+
+ENV CPPCHECK_DIR=/tmp/cppcheck
+COPY tests/misra/install.sh /tmp/
+RUN /tmp/install.sh
 
-ENV PYTHONPATH /tmp:$PYTHONPATH
+RUN git config --global --add safe.directory /tmp/openpilot/panda
+RUN cd /tmp && \
+    git clone https://github.com/commaai/openpilot.git tmppilot || true && \
+    cd /tmp/tmppilot && \
+    git fetch origin $OPENPILOT_REF && \
+    git checkout $OPENPILOT_REF && \
+    git submodule update --init cereal opendbc rednose_repo && \
+    git -C opendbc fetch && \
+    git -C opendbc checkout $OPENDBC_REF && \
+    git -C opendbc reset --hard HEAD && \
+    git -C opendbc clean -xfd && \
+    mkdir /tmp/openpilot && \
+    cp -pR SConstruct site_scons/ tools/ selfdrive/ system/ common/ cereal/ opendbc/ rednose/ third_party/ /tmp/openpilot && \
+    rm -rf /tmp/openpilot/panda && \
+    rm -rf /tmp/tmppilot
 
-RUN cd /tmp && git clone https://github.com/commaai/panda_jungle.git && \
+RUN cd /tmp/openpilot && \
+    git clone https://github.com/commaai/panda_jungle.git && \
     cd panda_jungle && \
     git fetch && \
-    git checkout 7b7197c605915ac34f3d62f314edd84e2e78a759
+    git checkout 7b7197c605915ac34f3d62f314edd84e2e78a759 && \
+    rm -rf .git/
 
-ADD ./panda.tar.gz /tmp/panda
+RUN cd /tmp/openpilot && \
+    pip install --no-cache-dir -r opendbc/requirements.txt && \
+    pip install --no-cache-dir --upgrade aenum lru-dict pycurl tenacity atomicwrites serial smbus2

Jenkinsfile

@@ -1,6 +1,22 @@
+def docker_run(String step_label, int timeout_mins, String cmd) {
+  timeout(time: timeout_mins, unit: 'MINUTES') {
+    sh script: "docker run --rm --privileged \
+          --env PARTIAL_TESTS=${env.PARTIAL_TESTS} \
+          --volume ${WORKSPACE.replace('/var/jenkins_home', '/data/docker/volumes/jenkins_jenkins-data/_data')}:/tmp/openpilot/panda \
+          --volume /dev/bus/usb:/dev/bus/usb \
+          --volume /var/run/dbus:/var/run/dbus \
+          --workdir /tmp/openpilot/panda \
+          --net host \
+          ${env.DOCKER_IMAGE_TAG} \
+          bash -c '${cmd}'", \
+        label: step_label
+  }
+}
+
 pipeline {
   agent any
   environment {
+    PARTIAL_TESTS = "${env.BRANCH_NAME == 'master' ? ' ' : '1'}"
     DOCKER_IMAGE_TAG = "panda:build-${env.GIT_COMMIT}"
   }
   stages {
@@ -13,69 +29,49 @@ pipeline {
           steps {
             timeout(time: 60, unit: 'MINUTES') {
               script {
-                sh 'git archive -v -o panda.tar.gz --format=tar.gz HEAD'
                 dockerImage = docker.build("${env.DOCKER_IMAGE_TAG}")
               }
             }
           }
         }
-        stage('reset hardware') {
+        stage('prep') {
           steps {
-            timeout(time: 10, unit: 'MINUTES') {
-              script {
-                sh "docker run --rm --privileged \
-                      --volume /dev/bus/usb:/dev/bus/usb \
-                      --volume /var/run/dbus:/var/run/dbus \
-                      --net host \
-                      ${env.DOCKER_IMAGE_TAG} \
-                      bash -c 'cd /tmp/panda && scons -j8 && python ./tests/ci_reset_hw.py'"
-              }
+            script {
+              docker_run("build", 1, "scons -j8")
+              docker_run("reset hardware", 3, "python ./tests/ci_reset_hw.py")
             }
           }
         }
         stage('pedal tests') {
           steps {
-            timeout(time: 10, unit: 'MINUTES') {
-              script {
-                sh "docker run --rm --privileged \
-                      --volume /dev/bus/usb:/dev/bus/usb \
-                      --volume /var/run/dbus:/var/run/dbus \
-                      --net host \
-                      ${env.DOCKER_IMAGE_TAG} \
-                      bash -c 'cd /tmp/panda && PEDAL_JUNGLE=058010800f51363038363036 python ./tests/pedal/test_pedal.py'"
-              }
+            script {
+              docker_run("test pedal", 1, "PEDAL_JUNGLE=058010800f51363038363036 python ./tests/pedal/test_pedal.py")
             }
           }
         }
         stage('HITL tests') {
           steps {
-            timeout(time: 30, unit: 'MINUTES') {
-              script {
-                sh "docker run --rm --privileged \
-                      --volume /dev/bus/usb:/dev/bus/usb \
-                      --volume /var/run/dbus:/var/run/dbus \
-                      --net host \
-                      ${env.DOCKER_IMAGE_TAG} \
-                      bash -c 'cd /tmp/panda && scons -j8 && PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE=\"1d0002000c51303136383232 2f002e000c51303136383232\" ./tests/automated/test.sh'"
-              }
+            script {
+              docker_run("HITL tests", 20, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/test.sh')
             }
           }
         }
         stage('CANFD tests') {
           steps {
-            timeout(time: 10, unit: 'MINUTES') {
-              script {
-                sh "docker run --rm --privileged \
-                      --volume /dev/bus/usb:/dev/bus/usb \
-                      --volume /var/run/dbus:/var/run/dbus \
-                      --net host \
-                      ${env.DOCKER_IMAGE_TAG} \
-                      bash -c 'cd /tmp/panda && scons -j8 && JUNGLE=058010800f51363038363036 H7_PANDAS_EXCLUDE=\"080021000c51303136383232 33000e001051393133353939\" ./tests/canfd/test_canfd.py'"
-              }
+            script {
+              docker_run("CANFD tets", 6, 'JUNGLE=058010800f51363038363036 H7_PANDAS_EXCLUDE="080021000c51303136383232 33000e001051393133353939" ./tests/canfd/test_canfd.py')
             }
           }
         }
       }
+
+      post {
+        always {
+          docker_run("git clean", 1, "git clean -xdff")
+          sh "rm -rf ${WORKSPACE}/* || true"
+          sh "rm -rf .* || true"
+        }
+      }
     }
   }
 }

requirements.txt

@@ -12,4 +12,5 @@ cffi==1.14.3
 crcmod
 pre-commit==2.13.0
 pylint==2.5.2
-scons==4.1.0.post1
+scons
+flaky

tests/black_white_loopback_test.py

@@ -23,7 +23,7 @@ def get_test_string():
 content_errors = 0
 
 def run_test(sleep_duration):
-  global counter, nonzero_bus_errors, zero_bus_errors, content_errors
+  global counter
 
   pandas = Panda.list()
   print(pandas)

tests/black_white_relay_endurance.py

@@ -23,7 +23,7 @@ def get_test_string():
 content_errors = 0
 
 def run_test(sleep_duration):
-  global counter, nonzero_bus_errors, zero_bus_errors, content_errors
+  global counter
 
   pandas = Panda.list()
   print(pandas)

tests/black_white_relay_test.py

@@ -22,7 +22,7 @@ def get_test_string():
 content_errors = 0
 
 def run_test(sleep_duration):
-  global counter, open_errors, closed_errors, content_errors
+  global counter, open_errors, closed_errors
 
   pandas = Panda.list()
   print(pandas)