ci: Add test workflow for integration/e2e (#1816)

.github/workflows/ci.yml

@@ -2,7 +2,8 @@
 name: CI
 on:
   push:
-    branches: master
+    branches:
+      - master
   pull_request:
 
 jobs:

.github/workflows/test.yml

@@ -0,0 +1,56 @@
+---
+name: Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  integration:
+    name: "Integration - VPP ${{ matrix.version }}"
+    runs-on: ubuntu-latest
+    env:
+      VPP_VERSION: ${{ matrix.version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ['2106', '2101', '2009', '2005']
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v2
+
+      - name: "Run tests"
+        run: |
+          make integration-tests
+
+  e2e:
+    name: "E2E - VPP ${{ matrix.version }}"
+    runs-on: ubuntu-latest
+    env:
+      VPP_VERSION: ${{ matrix.version }}
+      GOTESTSUM_JUNITFILE: /testreport/e2e.xml
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ['2106', '2101', '2009', '2005']
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v2
+
+      - name: "Dump Info"
+        run: |
+          env | sort
+          ls -Al /sys/module
+
+      - name: "Run Tests"
+        run: |
+          make e2e-tests
+
+      - name: "Publish Test Report"
+        uses: mikepenz/action-junit-report@v2
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: './tests/e2e/reports/*.xml'

.yamllint.yml

@@ -1,6 +1,10 @@
 ---
 extends: default
 
+ignore: |
+  /k8s/
+  /.travis.yml
+
 rules:
   # 80 chars should be enough, but don't fail if a line is longer
   line-length:

Makefile

@@ -263,7 +263,7 @@ gotestsumcmd := $(shell command -v gotestsum 2> /dev/null)
 
 test-tools: ## install test tools
 ifndef gotestsumcmd
-	go get -v gotest.tools/gotestsum
+	go get gotest.tools/gotestsum
 endif
 	@env CGO_ENABLED=0 go build -ldflags="-s -w" -o $(BUILD_DIR)/test2json cmd/test2json
 

docker/dev/Dockerfile

@@ -110,7 +110,7 @@ COPY ./docker/dev/init_hook.sh /usr/bin/
 # handle differences in vpp.conf which are between supported VPP versions
 ARG VPP_VERSION
 COPY ./docker/dev/legacy-nat.conf /tmp/legacy-nat.conf
-RUN if [[ "$VPP_VERSION" < "2009" ]]; then \
+RUN if [ "$VPP_VERSION" -le 2009 ]; then \
 		cat /tmp/legacy-nat.conf >> /etc/vpp/vpp.conf; \
 	fi; \
 	rm /tmp/legacy-nat.conf

docker/prod/Dockerfile

@@ -42,7 +42,7 @@ COPY init_hook.sh /usr/bin/
 # handle differences in vpp.conf which are between supported VPP versions
 ARG VPP_VERSION
 COPY legacy-nat.conf /tmp/legacy-nat.conf
-RUN if [[ "$VPP_VERSION" < "2009" ]]; then \
+RUN if [ "$VPP_VERSION" -le 2009 ]; then \
 		cat /tmp/legacy-nat.conf >> /etc/vpp/vpp.conf; \
 	fi; \
 	rm /tmp/legacy-nat.conf

tests/e2e/080_vrf_test.go

@@ -16,7 +16,6 @@ package e2e
 
 import (
 	"context"
-	"os"
 	"testing"
 	"time"
 
@@ -53,10 +52,8 @@ import (
 //    +-------------------------------+  +-------------------------------+
 //
 func TestVRFsWithSameSubnets(t *testing.T) {
-	if os.Getenv("TRAVIS") != "" {
-		// VRFs are seemingly not supported on Ubuntu Xenial, which is used in Travis CI to run the tests.
-		// TODO: remove `skip` once we upgrade to Ubuntu Bionic or newer
-		t.Skip("skip for travis")
+	if !supportsLinuxVRF() {
+		t.Skip("Linux VRFs are not supported")
 	}
 
 	ctx := Setup(t)
@@ -277,10 +274,8 @@ func TestVRFsWithSameSubnets(t *testing.T) {
 //    +-------------------------------+            +-------------------------------+
 //
 func TestVRFRoutes(t *testing.T) {
-	if os.Getenv("TRAVIS") != "" {
-		// VRFs are seemingly not supported on Ubuntu Xenial, which is used in Travis CI to run the tests.
-		// TODO: remove `skip` once we upgrade to Ubuntu Bionic or newer
-		t.Skip("skip for travis")
+	if !supportsLinuxVRF() {
+		t.Skip("Linux VRFs are not supported")
 	}
 
 	ctx := Setup(t)
@@ -488,10 +483,8 @@ func TestVRFRoutes(t *testing.T) {
 
 // Test VRF created externally (i.e. not by the agent).
 func TestExistingLinuxVRF(t *testing.T) {
-	if os.Getenv("TRAVIS") != "" {
-		// VRFs are seemingly not supported on Ubuntu Xenial, which is used in Travis CI to run the tests.
-		// TODO: remove `skip` once we upgrade to Ubuntu Bionic or newer
-		t.Skip("skip for travis")
+	if !supportsLinuxVRF() {
+		t.Skip("Linux VRFs are not supported")
 	}
 
 	ctx := Setup(t)

tests/e2e/13_dhcp_proxy_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	. "github.com/onsi/gomega"
+
 	"go.ligato.io/vpp-agent/v3/proto/ligato/kvscheduler"
 	linux_interfaces "go.ligato.io/vpp-agent/v3/proto/ligato/linux/interfaces"
 	linux_namespace "go.ligato.io/vpp-agent/v3/proto/ligato/linux/namespace"
@@ -28,6 +29,10 @@ import (
 )
 
 func TestDhcpProxy(t *testing.T) {
+	if !supportsLinuxVRF() {
+		t.Skip("Linux VRFs are not supported")
+	}
+
 	ctx := Setup(t)
 	defer ctx.Teardown()
 

tests/e2e/e2e.go

@@ -60,9 +60,9 @@ const (
 
 	// VPP input nodes for packet tracing (uncomment when needed)
 	tapv2InputNode = "virtio-input"
-	//tapv1InputNode    = "tapcli-rx"
-	//afPacketInputNode = "af-packet-input"
-	//memifInputNode    = "memif-input"
+	// tapv1InputNode    = "tapcli-rx"
+	// afPacketInputNode = "af-packet-input"
+	// memifInputNode    = "memif-input"
 )
 
 // TestCtx represents data context fur currently running test
@@ -143,7 +143,7 @@ func NewTest(t *testing.T) *TestCtx {
 	RegisterTestingT(t)
 	// TODO: Do not use global test registration.
 	//  It is now deprecated and you should use NewWithT() instead.
-	//g := NewWithT(t)
+	// g := NewWithT(t)
 
 	logrus.Debugf("Environ:\n%v", strings.Join(os.Environ(), "\n"))
 
@@ -283,7 +283,7 @@ func SetupVPPAgent(testCtx *TestCtx, opts ...AgentOptModifier) {
 // AgentInstanceName provides instance name of VPP-Agent that is created by setup by default. This name is
 // used i.e. in ETCD key prefix.
 func AgentInstanceName(testCtx *TestCtx) string {
-	//TODO API boundaries becomes blurry as tests and support structures are in the same package and there
+	// TODO API boundaries becomes blurry as tests and support structures are in the same package and there
 	// is strong temptation to misuse it and create an unmaintainable dependency mesh -> create different
 	// package for test supporting files (setup/teardown/util stuff) and define clear boundaries
 	if testCtx.Agent != nil {
@@ -773,3 +773,22 @@ func (test *TestCtx) checkAgentReady() error {
 	}
 	return nil
 }
+
+func supportsLinuxVRF() bool {
+	if os.Getenv("GITHUB_WORKFLOW") != "" {
+		// Linux VRFs are not enabled by default in the github workflow runners
+		// Notes:
+		// generally, run this to check system support for VRFs:
+		//  	modinfo vrf
+		// in the container, you can check if kernel module for VRFs is loaded:
+		//  	ls /sys/module/vrf
+		// TODO: figure out how to enable support for linux VRFs
+		return false
+	}
+	if os.Getenv("TRAVIS") != "" {
+		// Linux VRFs are seemingly not supported on Ubuntu Xenial, which is used in Travis CI to run the tests.
+		// TODO: remove once we upgrade to Ubuntu Bionic or newer
+		return false
+	}
+	return true
+}

tests/e2e/run_e2e.sh

@@ -10,7 +10,9 @@ echo "Preparing e2e tests.."
 export VPP_AGENT="${VPP_AGENT:-ligato/vpp-agent:latest}"
 export VPP_AGENT_CUSTOM="vppagent.test.ligato.io:custom"
 export TESTDATA_DIR="$SCRIPT_DIR/resources"
+export TESTREPORT_DIR="${TESTREPORT_DIR:-$SCRIPT_DIR/reports}"
 export GOTESTSUM_FORMAT="${GOTESTSUM_FORMAT:-testname}"
+export GOTESTSUM_JUNITFILE="${GOTESTSUM_JUNITFILE:-}"
 export DOCKER_BUILDKIT=1
 
 testname="vpp-agent-e2e-test"
@@ -42,8 +44,8 @@ docker build \
     ./tests/e2e
 
 # Build custom VPP-Agent image (needed in some tests)
-docker run -d -e ETCD_CONFIG="disabled" --name customVPPAgent ${VPP_AGENT}
-docker exec -it customVPPAgent sh -c "apt-get update && apt-get install -y iptables"
+docker run -d -e ETCD_CONFIG=disabled --name customVPPAgent ${VPP_AGENT}
+docker exec -i customVPPAgent sh -c "apt-get update && apt-get install -y iptables"
 docker commit customVPPAgent ${VPP_AGENT_CUSTOM}
 docker rm -f customVPPAgent
 
@@ -115,6 +117,8 @@ else
 	exit $res
 fi
 
+mkdir -vp "${TESTREPORT_DIR}"
+
 vppver=$(docker run --rm -i "$VPP_AGENT" dpkg-query -f '${Version}' -W vpp)
 
 echo "=========================================================================="
@@ -128,19 +132,22 @@ echo "--------------------------------------------------------------------------
 
 # Run e2e tests
 #if run_e2e ${args[@]:-}
-if docker run -it \
+if docker run -i \
 	--name "${testname}" \
 	--pid=host \
 	--privileged \
 	--label io.ligato.vpp-agent.testsuite=e2e \
 	--label io.ligato.vpp-agent.testname="${testname}" \
+	--volume "${TESTREPORT_DIR}":/testreport \
 	--volume "${TESTDATA_DIR}":/testdata:ro \
 	--volume /var/run/docker.sock:/var/run/docker.sock \
 	--volume "${sharevolumename}":/test-share \
 	--env TESTDATA_DIR \
 	--env INITIAL_LOGLVL \
 	--env VPP_AGENT \
 	--env GOTESTSUM_FORMAT \
+	--env GOTESTSUM_JUNITFILE \
+	--env GITHUB_WORKFLOW \
 	${DOCKER_ARGS-} \
 	"${imgname}" ${args[@]:-}
 then

tests/integration/Dockerfile.integration

@@ -22,9 +22,6 @@ RUN apt-get update && apt-get install -yq \
 # - test2json
 # - gotestsum
 
-#RUN env CGO_ENABLED=0 \
-# 	go build -ldflags='-s -w -extldflags "-static"' -o /usr/local/bin/test2json cmd/test2json
-
 COPY --from=build /test2json /usr/local/bin/test2json
 
 ARG GOTESTSUM_VERSION=0.6.0

tests/integration/entrypoint.sh

@@ -2,6 +2,6 @@
 set -e
 
 exec \
-  gotestsum --raw-command --format testname -- \
+  gotestsum --raw-command -- \
   test2json -t -p "integration" \
   /integration.test -test.v "$@"