CI: set and verify DESIRED_NETWORK (netavark, cni)

We have CI tests running in netavark mode when CNI is desired.
Add a new .cirrus.yml envariable, CI_DESIRED_NETWORK, which
we then force-check in e2e and system tests. Simple copy/paste
of #14912 (the RUNTIME check) with manual s/RUNTIME/NETWORK/
and other minor changes.

Signed-off-by: Ed Santiago <santiago@redhat.com>

.cirrus.yml

@@ -103,16 +103,19 @@ build_task:
               CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
               # ID for re-use of build output
               CI_DESIRED_RUNTIME: crun
+              CI_DESIRED_NETWORK: netavark
         - env: &priorfedora_envvars
               DISTRO_NV: ${PRIOR_FEDORA_NAME}
               VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
               CI_DESIRED_RUNTIME: crun
+              CI_DESIRED_NETWORK: cni
         #- env: &ubuntu_envvars
         #      DISTRO_NV: ${UBUNTU_NAME}
         #      VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME}
         #      CTR_FQIN: ${UBUNTU_CONTAINER_FQIN}
         #      CI_DESIRED_RUNTIME: runc
+        #      CI_DESIRED_NETWORK: whatever
     env:
         TEST_FLAVOR: build
     # NOTE: The default way Cirrus-CI clones is *NOT* compatible with
@@ -192,6 +195,7 @@ build_aarch64_task:
         VM_IMAGE_NAME: ${FEDORA_AARCH64_AMI}
         CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
         CI_DESIRED_RUNTIME: crun
+        CI_DESIRED_NETWORK: netavark
         TEST_FLAVOR: build
     clone_script: *full_clone
     prebuild_script: *prebuild
@@ -591,11 +595,13 @@ container_integration_test_task:
               VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
               CI_DESIRED_RUNTIME: crun
+              CI_DESIRED_NETWORK: netavark
         - env:
               DISTRO_NV: ${PRIOR_FEDORA_NAME}
               VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
               CI_DESIRED_RUNTIME: crun
+              CI_DESIRED_NETWORK: cni
     gce_instance: *standardvm
     timeout_in: 90m
     env:
@@ -650,6 +656,7 @@ podman_machine_task:
       PRIV_NAME: "rootless"  # intended use-case
       DISTRO_NV: "${FEDORA_NAME}"
       VM_IMAGE_NAME: "${FEDORA_AMI}"
+      CI_DESIRED_NETWORK: netavark
     clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
@@ -675,6 +682,7 @@ podman_machine_aarch64_task:
         PRIV_NAME: "rootless"  # intended use-case
         DISTRO_NV: "${FEDORA_AARCH64_NAME}"
         VM_IMAGE_NAME: "${FEDORA_AARCH64_AMI}"
+        CI_DESIRED_NETWORK: netavark
     clone_script: *get_gosrc_aarch64
     setup_script: *setup
     main_script: *main
@@ -760,6 +768,7 @@ rootless_remote_system_test_task:
               VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
               CI_DESIRED_RUNTIME: crun
+              CI_DESIRED_NETWORK: netavark
     <<: *local_system_test_task
     alias: rootless_remote_system_test
     depends_on:
@@ -822,6 +831,7 @@ buildah_bud_test_task:
         # Not used here, is used in other tasks
         VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
         CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+        CI_DESIRED_NETWORK: netavark
     matrix:
         - env:
             PODBIN_NAME: podman
@@ -874,10 +884,13 @@ upgrade_test_task:
     matrix:
         - env:
               PODMAN_UPGRADE_FROM: v2.1.1
+              CI_DESIRED_NETWORK: cni
         - env:
               PODMAN_UPGRADE_FROM: v3.1.2
+              CI_DESIRED_NETWORK: cni
         - env:
               PODMAN_UPGRADE_FROM: v3.4.4
+              CI_DESIRED_NETWORK: cni
     gce_instance: *standardvm
     env:
         TEST_FLAVOR: upgrade_test

contrib/cirrus/lib.sh

@@ -214,6 +214,9 @@ use_cni() {
     msg "Force-removing netavark and aardvark-dns"
     # Other packages depend on nv/av, but we're testing with podman
     # binaries built from source, so it's safe to ignore these deps.
+    #
+    # FIXME FIXME FIXME: if/when we bring back Ubuntu (or use Debian),
+    #       someone will have to conditionalize these rpm/dnf commands
     rpm -e --nodeps netavark aardvark-dns
     msg "Installing default CNI configuration"
     dnf install -y $PACKAGE_DOWNLOAD_DIR/podman-plugins*
@@ -236,9 +239,9 @@ use_netavark() {
     export NETWORK_BACKEND=netavark  # needed for install_test_configs()
     msg "Removing any/all CNI configuration"
     rm -rvf /etc/cni/net.d/*
-    # N/B: The netavark/aardvark-dns packages are still installed and
-    # available.  This is on purpose, since CI needs to verify the
-    # selection mechanisms are functional when both are available.
+    # N/B: The CNI packages are still installed and available. This is
+    # on purpose, since CI needs to verify the selection mechanisms are
+    # functional when both are available.
 }
 
 # Remove all files provided by the distro version of podman.

contrib/cirrus/setup_environment.sh

@@ -125,26 +125,19 @@ case "$OS_RELEASE_ID" in
             msg "Enabling container_manage_cgroup"
             setsebool container_manage_cgroup true
         fi
-
-        # For the latest Fedora CI VM images, netavark/aardvark is the
-        # intended networking stack for podman.  All previous VM images
-        # should use CNI networking.  Upgrading from one to the other is
-        # not supported at this time.  The only exception in CI is
-        # the "upgrade tests" which must always use CNI.
-        #
-        # OS_RELEASE_VER is defined by automation-library
-        # shellcheck disable=SC2154
-        if [[ "$DISTRO_NV" != "$PRIOR_FEDORA_NAME" ]] && \
-           [[ "$TEST_FLAVOR" != "upgrade_test" ]];
-        then
-            use_netavark
-        else # Fedora N-1 or upgrade testing.
-            use_cni
-        fi
         ;;
     *) die_unknown OS_RELEASE_ID
 esac
 
+# Networking: force CNI or Netavark as requested in .cirrus.yml
+# (this variable is mandatory).
+# shellcheck disable=SC2154
+case "$CI_DESIRED_NETWORK" in
+    netavark)   use_netavark ;;
+    cni)        use_cni ;;
+    *)          die_unknown CI_DESIRED_NETWORK ;;
+esac
+
 # Required to be defined by caller: The environment where primary testing happens
 # shellcheck disable=SC2154
 case "$TEST_ENVIRON" in
@@ -196,6 +189,7 @@ esac
 # Required to be defined by caller: Are we testing as root or a regular user
 case "$PRIV_NAME" in
     root)
+        # shellcheck disable=SC2154
         if [[ "$TEST_FLAVOR" = "sys" || "$TEST_FLAVOR" = "apiv2" ]]; then
             # Used in local image-scp testing
             setup_rootless
@@ -212,6 +206,7 @@ case "$PRIV_NAME" in
     *) die_unknown PRIV_NAME
 esac
 
+# shellcheck disable=SC2154
 if [[ -n "$ROOTLESS_USER" ]]; then
     echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
     echo "ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment

test/e2e/info_test.go

@@ -166,4 +166,19 @@ var _ = Describe("Podman Info", func() {
 		Expect(session).To(Exit(0))
 		Expect(session.OutputToString()).To(Equal(want))
 	})
+
+	It("Podman info: check desired network backend", func() {
+		// defined in .cirrus.yml
+		want := os.Getenv("CI_DESIRED_NETWORK")
+		if want == "" {
+			if os.Getenv("CIRRUS_CI") == "" {
+				Skip("CI_DESIRED_NETWORK is not set--this is OK because we're not running under Cirrus")
+			}
+			Fail("CIRRUS_CI is set, but CI_DESIRED_NETWORK is not! See #16389")
+		}
+		session := podmanTest.Podman([]string{"info", "--format", "{{.Host.NetworkBackend}}"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(session.OutputToString()).To(Equal(want))
+	})
 })

test/system/005-info.bats

@@ -75,6 +75,24 @@ host.slirp4netns.executable | $expr_path
     is "$output" "$CI_DESIRED_RUNTIME" "CI_DESIRED_RUNTIME (from .cirrus.yml)"
 }
 
+@test "podman info - confirm desired network backend" {
+    if [[ -z "$CI_DESIRED_NETWORK" ]]; then
+        # When running in Cirrus, CI_DESIRED_NETWORK *must* be defined
+        # in .cirrus.yml so we can double-check that all CI VMs are
+        # using netavark or cni as desired.
+        if [[ -n "$CIRRUS_CI" ]]; then
+            die "CIRRUS_CI is set, but CI_DESIRED_NETWORK is not! See #16389"
+        fi
+
+        # Not running under Cirrus (e.g., gating tests, or dev laptop).
+        # Totally OK to skip this test.
+        skip "CI_DESIRED_NETWORK is unset--OK, because we're not in Cirrus"
+    fi
+
+    run_podman info --format '{{.Host.NetworkBackend}}'
+    is "$output" "$CI_DESIRED_NETWORK" "CI_DESIRED_NETWORK (from .cirrus.yml)"
+}
+
 # 2021-04-06 discussed in watercooler: RHEL must never use crun, even if
 # using cgroups v2.
 @test "podman info - RHEL8 must use runc" {