CI: test overlay and vfs

We're only testing vfs in CI. That's bad. containers#18822 tried to
remedy that but I'm pretty sure it's a NOP.

Here we introduce CI_DESIRED_STORAGE, to be set in .cirrus.yml
in the same vein as all the other CI_DESIRED_X. Since it's 2023
we default to overlay, testing vfs only in priorfedora.

Skip the "split imagestore" test under overlay (containers#19748)

Signed-off-by: Ed Santiago <santiago@redhat.com>

.cirrus.yml

@@ -59,6 +59,7 @@ env:
     VM_IMAGE_NAME:           # One of the "Google-cloud VM Images" (above)
     CTR_FQIN:                # One of the "Container FQIN's" (above)
     CI_DESIRED_DATABASE: boltdb # One of "", "sqlite", "boltdb"
+    CI_DESIRED_STORAGE: overlay # overlay or vfs
 
     # Curl-command prefix for downloading task artifacts, simply add the
     # the url-encoded task name, artifact name, and path as a suffix.
@@ -120,6 +121,7 @@ build_task:
               CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
               CI_DESIRED_RUNTIME: crun
               CI_DESIRED_NETWORK: cni
+              CI_DESIRED_STORAGE: vfs
               # Catch invalid "TMPDIR == /tmp" assumptions; PR #19281
               TMPDIR: /var/tmp
         - env:

contrib/cirrus/lib.sh

@@ -95,7 +95,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
 # contexts, such as host->container or root->rootless user
 #
 # List of envariables which must be EXACT matches
-PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS|EC2_INST_TYPE|PODMAN_DB'
+PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS|EC2_INST_TYPE|PODMAN_DB|STORAGE_FS'
 
 # List of envariable patterns which must match AT THE BEGINNING of the name.
 PASSTHROUGH_ENV_ATSTART='CI|LANG|LC_|TEST'

contrib/cirrus/setup_environment.sh

@@ -103,21 +103,19 @@ esac
 # shellcheck disable=SC2154
 printf "[engine]\ndatabase_backend=\"$CI_DESIRED_DATABASE\"\n" > /etc/containers/containers.conf.d/92-db.conf
 
-# For debian envs pre-configure storage driver as overlay.
-# See: Discussion here https://github.com/containers/podman/pull/18510#discussion_r1189812306
-# for more details.
-# TODO: remove this once all CI VM have newer buildah version. (i.e where buildah
-# does not defaults to using `vfs` as storage driver)
+# Force the requested storage driver; this applies only to system tests.
 # shellcheck disable=SC2154
-if [[ "$OS_RELEASE_ID" == "debian" ]]; then
-    showrun echo "conditional setup for debian"
-    conf=/etc/containers/storage.conf
-    if [[ -e $conf ]]; then
-        die "FATAL! INTERNAL ERROR! Cannot override $conf"
-    fi
-    msg "Overriding $conf, setting overlay (was: $buildah_storage)"
-    printf '[storage]\ndriver = "overlay"\nrunroot = "/run/containers/storage"\ngraphroot = "/var/lib/containers/storage"\n' >$conf
+showrun echo "Setting up containers.conf for CI_DESIRED_STORAGE=$CI_DESIRED_STORAGE"
+conf=/etc/containers/storage.conf
+if [[ -e $conf ]]; then
+    die "FATAL! INTERNAL ERROR! Cannot override $conf"
 fi
+cat <<EOF >$conf
+[storage]
+driver = "$CI_DESIRED_STORAGE"
+runroot = "/run/containers/storage"
+graphroot = "/var/lib/containers/storage"
+EOF
 
 if ((CONTAINER==0)); then  # Not yet running inside a container
     showrun echo "conditional setup for CONTAINER == 0"
@@ -204,6 +202,10 @@ case "$CI_DESIRED_DATABASE" in
         ;;
 esac
 
+# shellcheck disable=SC2154
+showrun echo "about to set up for CI_DESIRED_STORAGE [=$CI_DESIRED_STORAGE]"
+echo "STORAGE_FS=$CI_DESIRED_STORAGE" >>/etc/ci_environment
+
 # Required to be defined by caller: The environment where primary testing happens
 # shellcheck disable=SC2154
 showrun echo "about to set up for TEST_ENVIRON [=$TEST_ENVIRON]"

test/e2e/info_test.go

@@ -185,6 +185,21 @@ var _ = Describe("Podman Info", func() {
 		Expect(session.OutputToString()).To(Equal(want))
 	})
 
+	It("Podman info: check desired storage driver", func() {
+		// defined in .cirrus.yml
+		want := os.Getenv("CI_DESIRED_STORAGE")
+		if want == "" {
+			if os.Getenv("CIRRUS_CI") == "" {
+				Skip("CI_DESIRED_STORAGE is not set--this is OK because we're not running under Cirrus")
+			}
+			Fail("CIRRUS_CI is set, but CI_DESIRED_STORAGE is not! See #20161")
+		}
+		session := podmanTest.Podman([]string{"info", "--format", "{{.Store.GraphDriverName}}"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(ExitCleanly())
+		Expect(session.OutputToString()).To(Equal(want), ".Store.GraphDriverName from podman info")
+	})
+
 	It("Podman info: check lock count", Serial, func() {
 		// This should not run on architectures and OSes that use the file locks backend.
 		// Which, for now, is Linux + RISCV and FreeBSD, neither of which are in CI - so

test/e2e/pull_test.go

@@ -66,6 +66,10 @@ var _ = Describe("Podman pull", func() {
 
 	It("podman pull and run on split imagestore", func() {
 		SkipIfRemote("podman-remote does not support setting external imagestore")
+		// Fails in run with "Error: creating container storage: creating read-write layer with ID "SHA1": Stat /.../root/overlay/SHA2/diff: ENOENT
+		if podmanTest.ImageCacheFS == "overlay" {
+			Skip("FIXME: #19748: test fails under overlayfs")
+		}
 		imgName := "splitstoretest"
 
 		// Make alpine write-able

test/system/001-basic.bats

@@ -53,6 +53,7 @@ function setup() {
         'Cgroups:{{.Host.CgroupsVersion}}+{{.Host.CgroupManager}}'
         'Net:{{.Host.NetworkBackend}}'
         'DB:{{.Host.DatabaseBackend}}'
+        'Store:{{.Store.GraphDriverName}}'
     )
     run_podman info --format "$(IFS='/' echo ${want[@]})"
     echo "# $output" >&3

test/system/005-info.bats

@@ -113,6 +113,23 @@ host.slirp4netns.executable | $expr_path
     is "$output" "$CI_DESIRED_DATABASE" "CI_DESIRED_DATABASE (from .cirrus.yml)"
 }
 
+@test "podman info - confirm desired storage driver" {
+    if [[ -z "$CI_DESIRED_STORAGE" ]]; then
+        # When running in Cirrus, CI_DESIRED_STORAGE *must* be defined
+        # in .cirrus.yml so we can double-check that all CI VMs are
+        # using overlay or vfs as desired.
+        if [[ -n "$CIRRUS_CI" ]]; then
+            die "CIRRUS_CI is set, but CI_DESIRED_STORAGE is not! See #20161"
+        fi
+
+        # Not running under Cirrus (e.g., gating tests, or dev laptop).
+        # Totally OK to skip this test.
+        skip "CI_DESIRED_STORAGE is unset--OK, because we're not in Cirrus"
+    fi
+
+    run_podman info --format '{{.Store.GraphDriverName}}'
+    is "$output" "$CI_DESIRED_STORAGE" "CI_DESIRED_STORAGE (from .cirrus.yml)"
+}
 
 # 2021-04-06 discussed in watercooler: RHEL must never use crun, even if
 # using cgroups v2.