CI: Use local cache registry

.cirrus.yml

@@ -33,7 +33,7 @@ env:
     DEBIAN_NAME: "debian-13"
 
     # Image identifiers
-    IMAGE_SUFFIX: "c20240702t180702z-f40f39d13"
+    IMAGE_SUFFIX: "c20240708t152000z-f40f39d13"
 
     # EC2 images
     FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}"

contrib/cirrus/get-local-registry-script

@@ -0,0 +1,163 @@
+#!/bin/bash
+#
+# get-local-registry-script - fetches local-registry script from automation_images repo
+#
+ME=$(basename $0)
+
+###############################################################################
+# BEGIN global settings
+
+SCRIPTNAME=cache_images/local-cache-registry
+
+REPO=containers/automation_images
+
+GITHUB_GRAPHQL=https://api.github.com/graphql
+
+# END   global settings
+###############################################################################
+
+usage="Usage: $ME [--help] [-v|--verbose] [IMG_SFX]
+
+--help          display usage message
+-v, --verbose   verbose output
+
+$ME fetches the $SCRIPTNAME tool
+from the github $REPO repo.
+If successful, the script will be left in cwd.
+
+It is up to you to invoke it:
+
+    \$ sudo ./$(basename $SCRIPTNAME) initialize
+
+This will set up a local registry and prepopulate it with
+images used in tests. Initialization may take tens of minutes
+on a slow connection.
+
+To use this registry in e2e or apiv2 tests:
+
+    export CI_USE_REGISTRY_CACHE=1
+
+To use in system tests, you need to overwrite /etc/containers/registries.conf
+because system tests, by definition, use system settings. This is left as
+an exercise for the reader.
+
+Reasons for doing this:
+  1) Consistency with CI environment
+  2) Speedier (and less flaky) tests
+"
+
+verbose=
+for i; do
+    value=`expr "$i" : '[^=]*=\(.*\)'`
+    case "$i" in
+    -h*|--help)	echo "$usage"; exit 0;;
+    -v|--verbose)	verbose=$i; shift;;
+    -*)	echo "$ME: unrecognized option $i" >&2
+	echo "$usage" >&2
+	exit 1;;
+    *)	break;;
+    esac
+done
+
+###############################################################################
+# BEGIN helper functions
+
+function die() {
+    echo "$ME: $*" >&2
+    exit 1
+}
+
+function get_imgsfx() {
+    test -e .cirrus.yml || die ".cirrus.yml does not exist; please run me from top of repo"
+
+    imgsfx=$(sed -ne 's/^ *IMAGE_SUFFIX:.*"c\(202.*\)"/\1/p' <.cirrus.yml)
+    if [[ -z "$imgsfx" ]]; then
+        die "Did not find 'IMAGE_SUFFIX:.*c202.*' in .cirrus.yml"
+    fi
+
+    expr "$imgsfx" : "^20[0-9]\{6\}t[0-9]\{6\}z-" &> /dev/null || die "IMAGE_SUFFIX '$imgsfx' (from .cirrus.yml) does not match expected YYYYMMDDtHHMMSS-* form"
+
+    echo $imgsfx
+}
+
+function get_script() {
+    local tag=$1
+
+    local tmpfile=$(mktemp --tmpdir --suffix=.sh $ME.fetched-script.XXXXXXX)
+
+    local url="https://raw.githubusercontent.com/${REPO}/$tag/$SCRIPTNAME"
+    test -n "$verbose" && echo "[ $url ]" >&2
+
+    curl -s -f -o $tmpfile $url
+    if [[ $? -eq 0 ]]; then
+        chmod 755 $tmpfile
+        mv -f $tmpfile $SCRIPTNAME
+        echo $SCRIPTNAME
+        return
+    fi
+    rm -f $tmpfile
+}
+
+# END   helper functions
+###############################################################################
+# BEGIN code
+
+if [[ -n "$1" ]]; then
+    imgsfx=$1
+    shift
+    if [[ -n "$*" ]]; then
+        die "Too many arguments; run $ME --help for help"
+    fi
+else
+    imgsfx=$(get_imgsfx)
+fi
+test -n "$verbose" && echo "[ imgsfx=$imgsfx ]" >&2
+
+# Easy case: this is a tagged automation_images build
+fetched_script=$(get_script $imgsfx)
+if [[ -n "$fetched_script" ]]; then
+    echo $fetched_script
+    exit 0
+fi
+
+# Nope, not a tagged build. Look for imgsfx in PR
+query="
+{
+    \"query\": \"{
+        search(query: \\\"repo:$REPO is:pr c$imgsfx\\\", type:ISSUE, first:10) {
+            edges {
+               node { ... on PullRequest {
+                       number
+                       title
+                       commits(last:30)  { nodes { commit { committedDate, oid } } }
+                       comments(last:30) { nodes { createdAt author { login } body } }
+                   }
+               }
+            }
+        }
+    }\"
+}
+"
+
+query_clean=$(tr -d \\012 <<<"$query")
+
+jsontmp=$(mktemp --tmpdir --suffix=.json $ME.graphql.XXXXXXX)
+
+curl -s -S -H "Authorization: bearer $GITHUB_TOKEN" -X POST --data "$query_clean" $GITHUB_GRAPHQL > $jsontmp
+
+# There should be one and exactly one comment from github-actions
+# that mentions the given imgsfx. Find its timestamp
+img_timestamp=$(jq -C -r '.data.search.edges.[].node.comments.nodes.[] | select(.author.login="github-actions") | select(.body | contains("c'${imgsfx}'")).createdAt' < $jsontmp)
+if [[ -z "$img_timestamp" ]]; then
+    die "Did not find a github-actions comment mentioning '$imgsfx'"
+fi
+
+# Some time prior to that comment, there was a commit pushed to github
+# that resulted in that build. Find that commit's SHA (OID).
+push_sha=$(jq -C -r '.data.search.edges.[].node.commits.nodes.[].commit | select(.committedDate < "'${img_timestamp}'").oid' < $jsontmp)
+
+# Got all the info we need. Clean up tmpfile
+rm -f $jsontmp
+
+# This must succeed.
+get_script $push_sha

contrib/cirrus/lib.sh

@@ -72,6 +72,9 @@ CIRRUS_CI="${CIRRUS_CI:-false}"
 CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
 CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-podman}
 
+# All CI jobs use a local registry
+export CI_USE_REGISTRY_CACHE=true
+
 # shellcheck disable=SC2154
 if [[ -n "$CIRRUS_PR" ]] && [[ -z "$PR_BASE_SHA" ]]; then
     # shellcheck disable=SC2154
@@ -193,7 +196,8 @@ setup_rootless() {
 
 install_test_configs() {
     msg "Installing ./test/registries.conf system-wide."
-    install -v -D -m 644 ./test/registries.conf /etc/containers/
+    # All CI VMs run with a local registry
+    install -v -D -m 644 ./test/registries-cached.conf /etc/containers/registries.conf
 }
 
 # Remove all files provided by the distro version of podman.

contrib/cirrus/runner.sh

@@ -124,6 +124,7 @@ exec_container() {
 
     # VM Images and Container images are built using (nearly) identical operations.
     set -x
+    env CONTAINERS_REGISTRIES_CONF=/dev/null bin/podman pull -q $CTR_FQIN
     # shellcheck disable=SC2154
     exec bin/podman run --rm --privileged --net=host --cgroupns=host \
         -v `mktemp -d -p /var/tmp`:/var/tmp:Z \

contrib/cirrus/setup_environment.sh

@@ -335,6 +335,18 @@ case "$PODBIN_NAME" in
     *) die_unknown PODBIN_NAME
 esac
 
+# As of July 2024, CI VMs come built-in with a registry.
+LCR=/var/cache/local-registry/local-cache-registry
+if [[ -x $LCR ]]; then
+    # Images in cache registry are prepopulated at the time
+    # VMs are built. If any PR adds a dependency on new images,
+    # those must be fetched now, at VM start time. This should
+    # be rare, and must be fixed in next automation_images build.
+    while read new_image; do
+        $LCR cache $new_image
+    done < <(grep '^[^#]' test/NEW-IMAGES || true)
+fi
+
 # Required to be defined by caller: The primary type of testing that will be performed
 # shellcheck disable=SC2154
 showrun echo "about to set up for TEST_FLAVOR [=$TEST_FLAVOR]"
@@ -426,6 +438,10 @@ case "$TEST_FLAVOR" in
         fi
         remove_packaged_podman_files
         showrun make install PREFIX=/usr ETCDIR=/etc
+        # machine-os image changes too frequently, can't be precached
+        # FIXME: I don't think we can use version.go, because of chicken-egg
+        # problem when that gets bumped. Ideas welcome.
+        $LCR cache podman/machine-os:5.2
         install_test_configs
         ;;
     swagger)

hack/podman-registry

@@ -7,7 +7,7 @@ ME=$(basename $0)
 ###############################################################################
 # BEGIN defaults
 
-PODMAN_REGISTRY_IMAGE=quay.io/libpod/registry:2.8
+PODMAN_REGISTRY_IMAGE=quay.io/libpod/registry:2.8.2
 
 PODMAN_REGISTRY_USER=
 PODMAN_REGISTRY_PASS=

pkg/bindings/test/images_test.go

@@ -177,7 +177,7 @@ var _ = Describe("Podman images", func() {
 
 		// Adding one more image. There Should be no errors in the response.
 		// And the count should be three now.
-		bt.Pull("testimage:20200929")
+		bt.Pull("testimage:20221018")
 		imageSummary, err = images.List(bt.conn, nil)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(len(imageSummary)).To(BeNumerically(">=", 2))

test/NEW-IMAGES

@@ -0,0 +1,14 @@
+#
+# As of July 2024, all CI VMs include a local registry prepopulated
+# with all container images used in tests:
+#
+#     https://github.com/containers/automation_images/pull/357
+#     https://github.com/containers/podman/pull/22726
+#
+# From time to time -- infrequently, we hope! -- existing images are
+# updated, or tests are added that require new images. Those must be
+# prefetched on each CI job, at least until new VMs are built. This
+# file contains those images.
+#
+# Format is one FQIN per line. Enumerate them below:
+#

test/apiv2/10-images.at

@@ -7,7 +7,9 @@
 podman pull -q $IMAGE
 
 t GET libpod/images/json 200 \
-  .[0].Id~[0-9a-f]\\{64\\}
+  length=1 \
+  .[0].Id~[0-9a-f]\\{64\\} \
+  .[0].Names[0]="$IMAGE"
 iid=$(jq -r '.[0].Id' <<<"$output")
 
 # Create an empty manifest and make sure it is not listed
@@ -18,7 +20,7 @@ t GET images/json 200 length=1
 t GET libpod/images/json 200 length=2
 
 t GET libpod/images/$iid/exists                     204
-t GET libpod/images/$PODMAN_TEST_IMAGE_NAME/exists  204
+t GET libpod/images/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG/exists  204
 t GET libpod/images/${iid}abcdef/exists  404 \
   .cause="failed to find image ${iid}abcdef"
 
@@ -51,11 +53,13 @@ t GET images/$iid/json 200 \
 t POST "images/create?fromImage=alpine" 200 .error~null .status~".*Download complete.*"
 t POST "libpod/images/pull?reference=alpine&compatMode=true" 200 .error~null .status~".*Download complete.*"
 
-t POST "images/create?fromImage=alpine&tag=latest" 200
+t POST "images/create?fromImage=alpine&tag=latest" 200 \
+  .status~"Already exists"
 
 # 10977 - handle platform parameter correctly
-t POST "images/create?fromImage=quay.io/libpod/testimage:20240123&platform=linux/arm64" 200
-t GET  "images/testimage:20240123/json" 200 \
+# THIS IMAGE MUST NOT BE THE SAME AS $IMAGE
+t POST "images/create?fromImage=quay.io/libpod/testimage:20221018&platform=linux/arm64" 200
+t GET  "images/testimage:20221018/json" 200 \
   .Architecture=arm64
 
 # Make sure that new images are pulled
@@ -78,34 +82,42 @@ t POST /images/create?fromImage=busybox:invalidtag123 404
 # Display the image history
 t GET libpod/images/nonesuch/history 404
 
-for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME; do
+for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG; do
   t GET libpod/images/$i/history 200 \
     .[0].Id=$iid \
     .[0].Created~[0-9]\\{10\\} \
     .[0].Tags[0]="$IMAGE" \
-    .[0].Size=0 \
-    .[0].Comment=
+    .[0].Size=1024 \
+    .[0].Comment="FROM localhost/interim-image:latest"
 done
 
-for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME; do
+for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG; do
   t GET images/$i/history 200 \
     .[0].Id="sha256:"$iid \
     .[0].Created~[0-9]\\{10\\} \
     .[0].Tags[0]="$IMAGE" \
-    .[0].Size=0 \
-    .[0].Comment=
+    .[0].Size=1024 \
+    .[0].Comment="FROM localhost/interim-image:latest"
 done
 
 # compat api pull image unauthorized message error
-t POST "/images/create?fromImage=quay.io/idonotexist/idonotexist:dummy" 401 \
-  .message="unauthorized: access to the requested resource is not authorized"
+# This depends on whether we're using local cache registry or real quay
+expect_code=401
+expect_msg="unauthorized: access to the requested resource is not authorized"
+if [[ -n "$CI_USE_REGISTRY_CACHE" ]]; then
+    # local registry has no auth, so it can return 404
+    expect_code=404
+    expect_msg="manifest unknown: manifest unknown"
+fi
+t POST "/images/create?fromImage=quay.io/idonotexist/idonotexist:dummy" $expect_code \
+  .message="$expect_msg"
 
 # Export an image on the local
 t GET libpod/images/nonesuch/get 404
 t GET libpod/images/$iid/get?format=foo 500
 t GET libpod/images/$PODMAN_TEST_IMAGE_NAME/get?compress=bar 400
 
-for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME; do
+for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG; do
   t GET "libpod/images/$i/get"                200 '[POSIX tar archive]'
   t GET "libpod/images/$i/get?compress=true"  200 '[POSIX tar archive]'
   t GET "libpod/images/$i/get?compress=false" 200 '[POSIX tar archive]'
@@ -296,10 +308,12 @@ t DELETE libpod/images/test:test 200
 # This sleep seems to avoid the race.
 # If it fails and begins to flake, investigate a retry loop.
 sleep 1
+# FIXME 2024-05-30 #22726: when running with a local cache registry, DELETE
+# sometimes produces 5-6 events instead of the desired only-one.
 t GET "libpod/events?stream=false&since=$START"  200  \
-  'select(.status | contains("remove")).Action=remove'
+  'select(.status | contains("remove")).Actor.Attributes.name~.*localhost/test:test'
 t GET "events?stream=false&since=$START"  200  \
-  'select(.status | contains("delete")).Action=delete'
+  'select(.status | contains("delete")).Actor.Attributes.name~.*localhost/test:test'
 
 # Test image removal with `noprune={true,false}`
 podman create --name c_test1 $IMAGE true