Merge pull request #5584 from edsantiago/ci-vms-with-local-registry

CI: use local registry

.cirrus.yml

@@ -32,7 +32,7 @@ env:
     DEBIAN_NAME: "debian-13"
 
     # Image identifiers
-    IMAGE_SUFFIX: "c20240620t153000z-f40f39d13"
+    IMAGE_SUFFIX: "c20240708t152000z-f40f39d13"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}"

contrib/cirrus/lib.sh

@@ -68,6 +68,10 @@ CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%d)}  # difficult to reliably
 CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-unknown$(date +%s)}  # must be short and unique enough
 CIRRUS_TASK_ID=${CIRRUS_BUILD_ID:-unknown$(date +%d)}   # to prevent state thrashing when
                                                         # debugging with `hack/get_ci_vm.sh`
+
+# All CI jobs use a local registry
+export CI_USE_REGISTRY_CACHE=true
+
 # Regex defining all CI-related env. vars. necessary for all possible
 # testing operations on all platforms and versions.  This is necessary
 # to avoid needlessly passing through global/system values across
@@ -92,8 +96,8 @@ PASSTHROUGH_ENV_RE="(^($PASSTHROUGH_ENV_EXACT)\$)|(^($PASSTHROUGH_ENV_ATSTART))|
 SECRET_ENV_RE='ACCOUNT|GC[EP]..|SSH|PASSWORD|SECRET|TOKEN'
 
 # FQINs needed for testing
-REGISTRY_FQIN=${REGISTRY_FQIN:-docker.io/library/registry}
-ALPINE_FQIN=${ALPINE_FQIN:-docker.io/library/alpine}
+REGISTRY_FQIN=${REGISTRY_FQIN:-quay.io/libpod/registry:2.8.2}
+ALPINE_FQIN=${ALPINE_FQIN:-quay.io/libpod/alpine}
 
 # for in-container testing
 IN_PODMAN_NAME="in_podman_$CIRRUS_TASK_ID"
@@ -189,7 +193,7 @@ in_podman() {
     done <<<"$(passthrough_envars)"
 
     showrun podman run -i --name="$IN_PODMAN_NAME" \
-                   --net="container:registry" \
+                   --net=host \
                    --privileged \
                    --cgroupns=host \
                    "${envargs[@]}" \

contrib/cirrus/setup.sh

@@ -87,6 +87,18 @@ echo "Configuring /etc/containers/registries.conf"
 mkdir -p /etc/containers
 echo -e "[registries.search]\nregistries = ['docker.io', 'registry.fedoraproject.org', 'quay.io']" | tee /etc/containers/registries.conf
 
+# As of July 2024, CI VMs come built-in with a registry.
+LCR=/var/cache/local-registry/local-cache-registry
+if [[ -x $LCR ]]; then
+    # Images in cache registry are prepopulated at the time
+    # VMs are built. If any PR adds a dependency on new images,
+    # those must be fetched now, at VM start time. This should
+    # be rare, and must be fixed in next automation_images build.
+    while read new_image; do
+        $LCR cache $new_image
+    done < <(grep '^[^#]' tests/NEW-IMAGES || true)
+fi
+
 show_env_vars
 
 if [[ -z "$CONTAINER" ]]; then

tests/NEW-IMAGES

@@ -0,0 +1,15 @@
+#
+# As of July 2024, all CI VMs include a local registry prepopulated
+# with all container images used in tests:
+#
+#     https://github.com/containers/automation_images/pull/357
+#     https://github.com/containers/podman/pull/22726
+#     https://github.com/containers/buildah/pull/5584
+#
+# From time to time -- infrequently, we hope! -- existing images are
+# updated, or tests are added that require new images. Those must be
+# prefetched on each CI job, at least until new VMs are built. This
+# file contains those images.
+#
+# Format is one FQIN per line. Enumerate them below:
+#

tests/bud.bats

@@ -4586,9 +4586,11 @@ EOM
 }
 
 @test "bud copy chown with newuser" {
-  _prefetch quay.io/fedora/fedora
+  _prefetch $SAFEIMAGE
   # Regression test for https://github.com/containers/buildah/issues/2192
-  run_buildah build $WITH_POLICY_JSON -t testctr -f $BUDFILES/copy-chown/Containerfile.chown_user $BUDFILES/copy-chown
+  run_buildah build $WITH_POLICY_JSON -t testctr \
+              --build-arg SAFEIMAGE=$SAFEIMAGE \
+              -f $BUDFILES/copy-chown/Containerfile.chown_user $BUDFILES/copy-chown
   expect_output --substring "myuser:myuser"
 }
 
@@ -6334,10 +6336,10 @@ _EOF
   expect_output --substring "world"
 }
 
-@test "bud-verify-if-we-dont-clean-prexisting-path" {
+@test "bud-verify-if-we-dont-clean-preexisting-path" {
   skip_if_no_runtime
   skip_if_in_container
-  _prefetch alpine debian
+  _prefetch alpine ubuntu
   run_buildah 1 build -t testbud $WITH_POLICY_JSON --secret id=secret-foo,src=$BUDFILES/verify-cleanup/secret1.txt -f $BUDFILES/verify-cleanup/Dockerfile $BUDFILES/verify-cleanup/
   expect_output --substring "hello"
   expect_output --substring "secrettext"

tests/bud/copy-chown/Containerfile.chown_user

@@ -1,7 +1,8 @@
-FROM quay.io/fedora/fedora
+ARG SAFEIMAGE
+FROM $SAFEIMAGE
 
 ENV MYUSER=myuser
-RUN useradd --create-home --home /"${MYUSER}" "${MYUSER}"
+RUN adduser -D -h /"${MYUSER}" "${MYUSER}"
 COPY --chown="${MYUSER}" ./copychown.txt /somewhere
 
 RUN stat -c "%U:%G" /somewhere

tests/bud/verify-cleanup/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine as builder
 RUN mkdir subdir
 COPY hey .
 
-FROM debian
+FROM ubuntu
 RUN  --mount=type=bind,source=.,dst=/tmp,z \
      --mount=type=tmpfs,dst=/var/tmp \
     cat /tmp/hey

tests/copy.bats

@@ -447,7 +447,7 @@ stuff/mystuff"
 @test "copy-preserving-extended-attributes" {
   createrandom ${TEST_SCRATCH_DIR}/randomfile
   # if we need to change which image we use, any image that can provide a working setattr/setcap/getfattr will do
-  image="quay.io/libpod/fedora-minimal:34"
+  image="quay.io/libpod/systemd-image:20240124"
   if ! which setfattr > /dev/null 2> /dev/null; then
     skip "setfattr not available, unable to check if it'll work in filesystem at ${TEST_SCRATCH_DIR}"
   fi

tests/digest.bats

@@ -30,11 +30,15 @@ fromreftest() {
 }
 
 @test "from-by-digest-s1" {
+  test -n "$CI_USE_REGISTRY_CACHE" && skip "Cannot test against local cache registry"
+
   skip_if_rootless_environment
   fromreftest quay.io/libpod/testdigest_v2s1@sha256:816563225d7baae4782653efc9410579341754fe32cbe20f7600b39fc37d8ec7
 }
 
 @test "from-by-digest-s1-a-discarded-layer" {
+  test -n "$CI_USE_REGISTRY_CACHE" && skip "Cannot test against local cache registry"
+
   skip_if_rootless_environment
   IMG=quay.io/libpod/testdigest_v2s1_with_dups@sha256:2c619fffbed29d8677e246798333e7d1b288333cb61c020575f6372c76fdbb52
 
@@ -54,6 +58,8 @@ fromreftest() {
 }
 
 @test "from-by-tag-s1" {
+  test -n "$CI_USE_REGISTRY_CACHE" && skip "Cannot test against local cache registry"
+
   skip_if_rootless_environment
   fromreftest quay.io/libpod/testdigest_v2s1:20200210
 }

tests/e2e/buildah_suite_test.go

@@ -103,6 +103,11 @@ func BuildahCreate(tempDir string) BuildAhTest {
 		storageOpts = fmt.Sprintf("--storage-driver %s", os.Getenv("STORAGE_DRIVER"))
 	}
 
+	registriesConfBase := "../../tests/registries"
+	if os.Getenv("CI_USE_REGISTRY_CACHE") != "" {
+		registriesConfBase += "-cached"
+	}
+
 	return BuildAhTest{
 		BuildAhBinary:  buildAhBinary,
 		RunRoot:        filepath.Join(tempDir, "runroot"),
@@ -111,7 +116,7 @@ func BuildahCreate(tempDir string) BuildAhTest {
 		ArtifactPath:   artifactDir,
 		TempDir:        tempDir,
 		SignaturePath:  "../../tests/policy.json",
-		RegistriesConf: "../../tests/registries.conf",
+		RegistriesConf: registriesConfBase + ".conf",
 	}
 }
 

tests/from.bats

@@ -110,22 +110,26 @@ load helpers
   run_buildah rm $output
   run_buildah rmi docker.io/alpine:latest
 
-  run_buildah from --quiet --pull=true $WITH_POLICY_JSON docker.io/centos:7
-  run_buildah rm $output
-  run_buildah rmi docker.io/centos:7
+  # FIXME FIXME FIXME: I don't see the point of these. Any reason not to delete?
+#  run_buildah from --quiet --pull=true $WITH_POLICY_JSON docker.io/centos:7
+#  run_buildah rm $output
+#  run_buildah rmi docker.io/centos:7
 
-  run_buildah from --quiet --pull=true $WITH_POLICY_JSON docker.io/centos:latest
-  run_buildah rm $output
-  run_buildah rmi docker.io/centos:latest
+#  run_buildah from --quiet --pull=true $WITH_POLICY_JSON docker.io/centos:latest
+#  run_buildah rm $output
+#  run_buildah rmi docker.io/centos:latest
 }
 
 @test "from the following transports: docker-archive, oci-archive, and dir" {
   _prefetch alpine
   run_buildah from --quiet --pull=true $WITH_POLICY_JSON alpine
   run_buildah rm $output
 
-  run_buildah from --quiet --pull=true $WITH_POLICY_JSON docker:latest
-  run_buildah rm $output
+  # #2205: The important thing here is differentiating 'docker:latest'
+  # (the image) from 'docker:/path' ('docker' as a protocol identifier).
+  # This is a parsing fix so we don't actually need to pull the image.
+  run_buildah 125 from --quiet --pull=false $WITH_POLICY_JSON docker:latest
+  assert "$output" = "Error: docker:latest: image not known"
 
   run_buildah push $WITH_POLICY_JSON alpine docker-archive:${TEST_SCRATCH_DIR}/docker-alp.tar:alpine
   run_buildah push $WITH_POLICY_JSON alpine    oci-archive:${TEST_SCRATCH_DIR}/oci-alp.tar:alpine

tests/helpers.bash

@@ -68,9 +68,17 @@ EOF
 
     # Common options for all buildah and podman invocations
     ROOTDIR_OPTS="--root ${TEST_SCRATCH_DIR}/root --runroot ${TEST_SCRATCH_DIR}/runroot --storage-driver ${STORAGE_DRIVER}"
-    BUILDAH_REGISTRY_OPTS="--registries-conf ${TEST_SOURCES}/registries.conf --registries-conf-dir ${TEST_SCRATCH_DIR}/registries.d --short-name-alias-conf ${TEST_SCRATCH_DIR}/cache/shortnames.conf"
-    COPY_REGISTRY_OPTS="--registries-conf ${TEST_SOURCES}/registries.conf --registries-conf-dir ${TEST_SCRATCH_DIR}/registries.d --short-name-alias-conf ${TEST_SCRATCH_DIR}/cache/shortnames.conf"
-    PODMAN_REGISTRY_OPTS="--registries-conf ${TEST_SOURCES}/registries.conf"
+
+    # When running in CI, use a local registry for all image pulls
+    local cached=
+    if [[ -n "$CI_USE_REGISTRY_CACHE" ]]; then
+        cached="-cached"
+    fi
+    regconfopt="--registries-conf ${TEST_SOURCES}/registries$cached.conf"
+    regconfdir="--registries-conf-dir ${TEST_SCRATCH_DIR}/registries.d"
+    BUILDAH_REGISTRY_OPTS="${regconfopt} ${regconfdir} --short-name-alias-conf ${TEST_SCRATCH_DIR}/cache/shortnames.conf"
+    COPY_REGISTRY_OPTS="${BUILDAH_REGISTRY_OPTS}"
+    PODMAN_REGISTRY_OPTS="${regconfopt}"
 }
 
 function starthttpd() {
@@ -676,7 +684,7 @@ function stop_git_daemon() {
 function start_registry() {
   local testuser="${1:-testuser}"
   local testpassword="${2:-testpassword}"
-  local REGISTRY_IMAGE=quay.io/libpod/registry:2.8
+  local REGISTRY_IMAGE=quay.io/libpod/registry:2.8.2
   local config='
 version: 0.1
 log:

tests/mkcw.bats

@@ -58,7 +58,14 @@ function mkcw_check_image() {
     skip "cryptsetup not found"
   fi
   _prefetch busybox
-  _prefetch bash
+  # The important thing we need from $SAFEIMAGE is that it have >1 layer.
+  # Per @nalind:
+  #     The error we were attempting to avoid was causing the disk image to lose
+  #     content from layers that weren't the last one (and as far as this test is
+  #     concerned, for images with one layer, the only layer is also the last layer),
+  #     and the presence of the second layer, empty as it is, means the image still
+  #     meets the test expectations.
+  _prefetch $SAFEIMAGE
   createrandom randomfile1
   createrandom randomfile2
 
@@ -67,8 +74,8 @@ function mkcw_check_image() {
   run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert --add-file randomfile1:/in-a-subdir/rnd1 busybox busybox-cw
   mkcw_check_image busybox-cw "" randomfile1:in-a-subdir/rnd1
   # image has multiple layers, check with all-upper-case TEE type name
-  run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert --add-file randomfile2:rnd2 bash bash-cw
-  mkcw_check_image bash-cw "" randomfile2:/rnd2
+  run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert --add-file randomfile2:rnd2 $SAFEIMAGE my-cw
+  mkcw_check_image my-cw "" randomfile2:/rnd2
 }
 
 @test "mkcw-commit" {
@@ -77,10 +84,10 @@ function mkcw_check_image() {
   if ! which cryptsetup > /dev/null 2> /dev/null ; then
     skip "cryptsetup not found"
   fi
-  _prefetch bash
+  _prefetch $SAFEIMAGE
 
   echo -n "mkcw commit" > "$TEST_SCRATCH_DIR"/key
-  run_buildah from bash
+  run_buildah from $SAFEIMAGE
   ctrID="$output"
   run_buildah commit --iidfile "$TEST_SCRATCH_DIR"/iid --cw type=SEV,ignore_attestation_errors,passphrase="mkcw commit" "$ctrID"
   mkcw_check_image $(cat "$TEST_SCRATCH_DIR"/iid)

tests/pull.bats

@@ -32,29 +32,30 @@ load helpers
   run_buildah 125 --registries-conf ${TEST_SOURCES}/registries.conf.block pull $WITH_POLICY_JSON docker.io/alpine
   expect_output --substring "registry docker.io is blocked in"
 
-  run_buildah --retry --registries-conf ${TEST_SOURCES}/registries.conf       pull $WITH_POLICY_JSON docker.io/alpine
+  run_buildah --retry pull $WITH_POLICY_JSON docker.io/alpine
 }
 
 @test "pull-from-registry" {
-  run_buildah --retry pull --registries-conf ${TEST_SOURCES}/registries.conf $WITH_POLICY_JSON busybox:glibc
-  run_buildah pull --registries-conf ${TEST_SOURCES}/registries.conf $WITH_POLICY_JSON busybox:latest
+  run_buildah --retry pull $WITH_POLICY_JSON busybox:glibc
+  run_buildah pull $WITH_POLICY_JSON busybox:latest
   run_buildah images --format "{{.Name}}:{{.Tag}}"
   expect_output --substring "busybox:glibc"
   expect_output --substring "busybox:latest"
   # We need to see if this file is created after first pull in at least one test
   [ -f ${TEST_SCRATCH_DIR}/root/defaultNetworkBackend ]
 
-  run_buildah --retry pull --registries-conf ${TEST_SOURCES}/registries.conf $WITH_POLICY_JSON quay.io/libpod/alpine_nginx:latest
+  run_buildah --retry pull $WITH_POLICY_JSON quay.io/libpod/alpine_nginx:latest
   run_buildah images --format "{{.Name}}:{{.Tag}}"
   expect_output --substring "alpine_nginx:latest"
 
   run_buildah rmi quay.io/libpod/alpine_nginx:latest
-  run_buildah --retry pull --registries-conf ${TEST_SOURCES}/registries.conf $WITH_POLICY_JSON quay.io/libpod/alpine_nginx
+  run_buildah --retry pull $WITH_POLICY_JSON quay.io/libpod/alpine_nginx
   run_buildah images --format "{{.Name}}:{{.Tag}}"
   expect_output --substring "alpine_nginx:latest"
 
-  run_buildah --retry pull --registries-conf ${TEST_SOURCES}/registries.conf $WITH_POLICY_JSON alpine@sha256:e9a2035f9d0d7cee1cdd445f5bfa0c5c646455ee26f14565dce23cf2d2de7570
-  run_buildah 125 pull --registries-conf ${TEST_SOURCES}/registries.conf $WITH_POLICY_JSON fakeimage/fortest
+  run_buildah --retry pull $WITH_POLICY_JSON alpine@sha256:634a8f35b5f16dcf4aaa0822adc0b1964bb786fca12f6831de8ddc45e5986a00
+
+  run_buildah 125 pull $WITH_POLICY_JSON fakeimage/fortest
   run_buildah images --format "{{.Name}}:{{.Tag}}"
   assert "$output" !~ "fakeimage/fortest" "fakeimage/fortest found in buildah images"
 }

tests/registries-cached.conf

@@ -0,0 +1,19 @@
+# Note that changing the order here may break tests.
+unqualified-search-registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org']
+
+[[registry]]
+# As of July 2024, all CI VMs come with a local registry prepopulated
+# with all images needed by tests.
+prefix="docker.io"
+location="127.0.0.1:60333"
+insecure=true
+
+[[registry]]
+prefix="quay.io"
+location="127.0.0.1:60333"
+insecure=true
+
+[[registry]]
+prefix="docker.io/library"
+location="127.0.0.1:60333/libpod"
+insecure=true

tests/run.bats

@@ -657,11 +657,14 @@ function configure_and_check_user() {
 	skip_if_in_container
 
 	${OCI} --version
-	_prefetch debian
+        # We use ubuntu image because it has no /etc/hosts file. This
+        # allows the fake_host test below to be an equality check,
+        # not a substring check.
+	_prefetch ubuntu
 
 	local hostname=h-$(random_string)
 
-	run_buildah from --quiet --pull=false $WITH_POLICY_JSON debian
+	run_buildah from --quiet --pull=false $WITH_POLICY_JSON ubuntu
 	cid=$output
 	run_buildah 125 run --network=bogus $cid cat /etc/hosts
 	expect_output --substring "unable to find network with name or ID bogus: network not found"
@@ -699,7 +702,7 @@ function configure_and_check_user() {
 	expect_output --substring ""
 	run_buildah rm -a
 
-	run_buildah from --quiet --pull=false $WITH_POLICY_JSON debian
+	run_buildah from --quiet --pull=false $WITH_POLICY_JSON ubuntu
 	cid=$output
 	run_buildah run --network=host --hostname $hostname $cid cat /etc/hosts
 	assert "$output" =~ "$ip[[:blank:]]$hostname"
@@ -715,7 +718,7 @@ function configure_and_check_user() {
 	assert "$output" =~ "$ip[[:blank:]]$hostname"
 	run_buildah rm -a
 
-	run_buildah from --quiet --pull=false $WITH_POLICY_JSON debian
+	run_buildah from --quiet --pull=false $WITH_POLICY_JSON ubuntu
 	cid=$output
 	run_buildah run --network=none $cid sh -c 'echo "110.110.110.0 fake_host" >> /etc/hosts; cat /etc/hosts'
 	expect_output "110.110.110.0 fake_host"