Merge pull request containers#8312 from cevich/new_ubuntu_images

Cirrus: Update VM Images; Both Fedora and Ubuntu "prior" flavors run with CGroupsV1 & runc

.cirrus.yml

@@ -26,11 +26,11 @@ env:
     ####
     FEDORA_NAME: "fedora-33"
     PRIOR_FEDORA_NAME: "fedora-32"
-    UBUNTU_NAME: "ubuntu-20"
-    PRIOR_UBUNTU_NAME: "ubuntu-19"
+    UBUNTU_NAME: "ubuntu-2010"
+    PRIOR_UBUNTU_NAME: "ubuntu-2004"
 
     # Google-cloud VM Images
-    IMAGE_SUFFIX: "c4704091098054656"
+    IMAGE_SUFFIX: "c6233039174893568"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"

contrib/cirrus/lib.sh

@@ -41,7 +41,7 @@ fi
 
 OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
 # GCE image-name compatible string representation of distribution _major_ version
-OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
+OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')"
 # Combined to ease soe usage
 OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 # This is normally set from .cirrus.yml but default is necessary when

contrib/cirrus/logcollector.sh

@@ -53,15 +53,15 @@ case $1 in
                     slirp4netns \
         )
         case $OS_RELEASE_ID in
-            fedora*)
+            fedora)
                 cat /etc/fedora-release
                 PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n'
                 PKG_NAMES+=(\
                     container-selinux \
                     libseccomp \
                 )
                 ;;
-            ubuntu*)
+            ubuntu)
                 cat /etc/issue
                 PKG_LST_CMD='dpkg-query --show --showformat=${Package}-${Version}-${Architecture}\n'
                 PKG_NAMES+=(\

contrib/cirrus/runner.sh

@@ -41,7 +41,7 @@ function _run_automation() {
     req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \
                  PODBIN_NAME PRIV_NAME DISTRO_NV CONTAINER USER HOME \
                  UID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \
-                 OS_RELEASE_VER CG_FS_TYPE
+                 CG_FS_TYPE
     bigto ooe.sh dnf install -y ShellCheck  # small/quick addition
     $SCRIPT_BASE/shellcheck.sh
 }
@@ -64,12 +64,6 @@ function _run_unit() {
 }
 
 function _run_apiv2() {
-    # TODO Remove once VM's with dependency
-    if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
-        dnf install -y python3-docker
-    else
-        apt-get -qq -y install python3-docker
-    fi
     make localapiv2 |& logformatter
 }
 

contrib/cirrus/setup_environment.sh

@@ -73,7 +73,9 @@ case "$CG_FS_TYPE" in
         if ((CONTAINER==0)); then
             warn "Forcing testing with runc instead of crun"
             if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then
-                echo "OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" >> /etc/ci_environment
+                # Need b/c using cri-o-runc package from OBS
+                echo "OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" \
+                    >> /etc/ci_environment
             else
                 echo "OCI_RUNTIME=runc" >> /etc/ci_environment
             fi
@@ -102,8 +104,8 @@ fi
 
 # Which distribution are we testing on.
 case "$OS_RELEASE_ID" in
-    ubuntu*) ;;
-    fedora*)
+    ubuntu) ;;
+    fedora)
         if ((CONTAINER==0)); then
             msg "Configuring / Expanding host storage."
             # VM is setup to allow flexibility in testing alternate storage.
@@ -123,10 +125,15 @@ esac
 # shellcheck disable=SC2154
 case "$TEST_ENVIRON" in
     host)
-        if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
-            # The e2e tests wrongly guess `--cgroup-manager cgroupfs`
+        # The e2e tests wrongly guess `--cgroup-manager` option
+        # shellcheck disable=SC2154
+        if [[ "$CG_FS_TYPE" == "cgroup2fs" ]] || [[ "$PRIV_NAME" == "root" ]]
+        then
             warn "Forcing CGROUP_MANAGER=systemd"
             echo "CGROUP_MANAGER=systemd" >> /etc/ci_environment
+        else
+            warn "Forcing CGROUP_MANAGER=cgroupfs"
+            echo "CGROUP_MANAGER=cgroupfs" >> /etc/ci_environment
         fi
         ;;
     container)
@@ -138,25 +145,21 @@ case "$TEST_ENVIRON" in
             modprobe ip6table_nat || :
             modprobe iptable_nat || :
         else
-            # The e2e tests wrongly guess `--cgroup-manager systemd`
             warn "Forcing CGROUP_MANAGER=cgroupfs"
             echo "CGROUP_MANAGER=cgroupfs" >> /etc/ci_environment
-        fi
-        ;;
-    *) die_unknown TEST_ENVIRON
-esac
 
-# Required to be defined by caller: Are we testing as root or a regular user
-# shellcheck disable=SC2154
-case "$PRIV_NAME" in
-    root)
-        if [[ "$TEST_ENVIRON" == "container" ]] && ((container)); then
             # There's no practical way to detect userns w/in a container
             # affected/related tests are sensitive to this variable.
             warn "Disabling usernamespace integration testing"
             echo "SKIP_USERNS=1" >> /etc/ci_environment
         fi
         ;;
+    *) die_unknown TEST_ENVIRON
+esac
+
+# Required to be defined by caller: Are we testing as root or a regular user
+case "$PRIV_NAME" in
+    root) ;;
     rootless)
         # Needs to exist for setup_rootless()
         ROOTLESS_USER="${ROOTLESS_USER:-some${RANDOM}dude}"

test/e2e/containers_conf_test.go

@@ -82,6 +82,7 @@ var _ = Describe("Podman run", func() {
 	})
 
 	It("podman Capabilities in containers.conf", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		cap := podmanTest.Podman([]string{"run", ALPINE, "grep", "CapEff", "/proc/self/status"})
 		cap.WaitWithDefaultTimeout()
 		Expect(cap.ExitCode()).To(Equal(0))
@@ -121,6 +122,7 @@ var _ = Describe("Podman run", func() {
 	})
 
 	verifyNSHandling := func(nspath, option string) {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		os.Setenv("CONTAINERS_CONF", "config/containers-ns.conf")
 		if IsRemote() {
 			podmanTest.RestartRemoteService()

test/e2e/generate_kube_test.go

@@ -471,6 +471,7 @@ var _ = Describe("Podman generate kube", func() {
 	})
 
 	It("podman generate kube multiple pods should fail", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		pod1 := podmanTest.Podman([]string{"run", "-dt", "--pod", "new:pod1", ALPINE, "top"})
 		pod1.WaitWithDefaultTimeout()
 		Expect(pod1.ExitCode()).To(Equal(0))

test/e2e/pod_infra_container_test.go

@@ -225,6 +225,7 @@ var _ = Describe("Podman pod create", func() {
 	})
 
 	It("podman pod container can override pod pid NS", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		session := podmanTest.Podman([]string{"pod", "create", "--share", "pid"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))

test/e2e/pod_kill_test.go

@@ -127,6 +127,7 @@ var _ = Describe("Podman pod kill", func() {
 	})
 
 	It("podman pod kill all", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		_, ec, podid := podmanTest.CreatePod("")
 		Expect(ec).To(Equal(0))
 

test/e2e/pod_ps_test.go

@@ -157,6 +157,7 @@ var _ = Describe("Podman ps", func() {
 	})
 
 	It("podman pod ps --ctr-names", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		_, ec, podid := podmanTest.CreatePod("")
 		Expect(ec).To(Equal(0))
 

test/e2e/pod_stats_test.go

@@ -17,8 +17,9 @@ var _ = Describe("Podman pod stats", func() {
 	)
 
 	BeforeEach(func() {
-		if os.Geteuid() != 0 {
-			SkipIfCgroupV2("--cgroup-manager=cgroupfs which doesn't work in rootless mode")
+		SkipIfRootless("Tests fail with both CGv1/2 + required --cgroup-manager=cgroupfs")
+		if isContainerized() {
+			SkipIfCgroupV1("All tests fail Error: unable to load cgroup at ...: cgroup deleted")
 		}
 
 		tempdir, err = CreateTempDirInTempDir()
@@ -176,7 +177,8 @@ var _ = Describe("Podman pod stats", func() {
 
 	It("podman stats on net=host post", func() {
 		// --net=host not supported for rootless pods at present
-		SkipIfRootlessCgroupsV1("Pause stats not supported in cgroups v1")
+		// problem with sysctls being passed to containers of the pod.
+		SkipIfCgroupV1("Bug: Error: sysctl net.ipv4.ping_group_range is not allowed in the hosts network namespace: OCI runtime error")
 		podName := "testPod"
 		podCreate := podmanTest.Podman([]string{"pod", "create", "--net=host", "--name", podName})
 		podCreate.WaitWithDefaultTimeout()

test/e2e/run_ns_test.go

@@ -35,6 +35,7 @@ var _ = Describe("Podman run ns", func() {
 	})
 
 	It("podman run pidns test", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		session := podmanTest.Podman([]string{"run", fedoraMinimal, "bash", "-c", "echo $$"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
@@ -105,6 +106,7 @@ var _ = Describe("Podman run ns", func() {
 	})
 
 	It("podman run --ipc=host --pid=host", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		cmd := exec.Command("ls", "-l", "/proc/self/ns/pid")
 		res, err := cmd.Output()
 		Expect(err).To(BeNil())

test/e2e/run_selinux_test.go

@@ -274,6 +274,7 @@ var _ = Describe("Podman run", func() {
 	})
 
 	It("podman test --pid=host", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		session := podmanTest.Podman([]string{"run", "--pid=host", ALPINE, "cat", "/proc/self/attr/current"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))

test/e2e/run_test.go

@@ -493,7 +493,9 @@ USER bin`
 				Skip("Kernel does not support blkio.weight")
 			}
 		}
-
+		if podmanTest.Host.Distribution == "ubuntu" {
+			Skip("Ubuntu <= 20.10 lacks BFQ scheduler")
+		}
 		if CGROUPSV2 {
 			// convert linearly from [10-1000] to [1-10000]
 			session := podmanTest.Podman([]string{"run", "--rm", "--blkio-weight=15", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.bfq.weight"})

test/e2e/toolbox_test.go

@@ -121,6 +121,7 @@ var _ = Describe("Toolbox-specific testing", func() {
 		if podmanTest.RemoteTest {
 			Skip("Shm size check does not work with a remote client")
 		}
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
 		var session *PodmanSessionIntegration
 		var cmd *exec.Cmd
 		var hostShmSize, containerShmSize int

test/system/600-completion.bats

@@ -8,6 +8,17 @@
 
 load helpers
 
+# Returns true if we are able to podman-pause
+function _can_pause() {
+    # Even though we're just trying completion, not an actual unpause,
+    # podman barfs with:
+    #    Error: unpause is not supported for cgroupv1 rootless containers
+    if is_rootless && is_cgroupsv1; then
+        return 1
+    fi
+    return 0
+}
+
 function check_shell_completion() {
     local count=0
 
@@ -70,8 +81,13 @@ function check_shell_completion() {
                     ;;
 
                 *CONTAINER*)
+                    # podman unpause fails early on rootless cgroupsv1
+                    if [[ $cmd = "unpause" ]] && ! _can_pause; then
+                        continue 2
+                    fi
+
                     run_completion "$@" $cmd "${extra_args[@]}" ""
-                    is "$output" ".*-$random_container_name${nl}" "Found expected container in suggestions"
+                    is "$output" ".*-$random_container_name${nl}" "Found expected container in suggestions for '$cmd'"
 
                     match=true
                     # resume
@@ -212,7 +228,9 @@ function _check_completion_end() {
     run_podman create --name created-$random_container_name $IMAGE
     run_podman run --name running-$random_container_name -d $IMAGE top
     run_podman run --name pause-$random_container_name -d $IMAGE top
-    run_podman pause pause-$random_container_name
+    if _can_pause; then
+        run_podman pause pause-$random_container_name
+    fi
     run_podman run --name exited-$random_container_name -d $IMAGE echo exited
 
     # create pods for each state