Rework image pull check in E2E tests (#196)

* Rework image pull check in E2E tests

* Address PR review feedback

test/common/common.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"slices"
 	"strings"
 	"time"
@@ -30,19 +29,25 @@ import (
 )
 
 const (
-	// DockerNginx1230ImageWithDigest corresponds to the nginx:1.23.0 image.
-	DockerNginx1230ImageWithDigest = "docker.io/library/nginx@sha256:db345982a2f2a4257c6f699a499feb1d79451a1305e8022f16456ddc3ad6b94c"
-	// DockerNginx1240ImageWithDigest corresponds to the nginx:1.24.0 image.
-	DockerNginx1240ImageWithDigest = "docker.io/library/nginx@sha256:066476749f229923b9de29cc9a0738ea2d45923b16a2b388449ea549673f97d8"
-	// DockerNginx1250ImageWithDigest corresponds to the nginx:1.25.0 image.
-	DockerNginx1250ImageWithDigest = "docker.io/library/nginx@sha256:b997b0db9c2bc0a2fb803ced5fb9ff3a757e54903a28ada3e50412cc3ab7822f"
-
-	// ArtifactRegistryNginx1176ImageWithDigest corresponds to the europe-docker.pkg.dev/gardener-project/releases/3rd/nginx:1.17.6 image (copy of docker.io/library/nginx:1.17.6).
-	ArtifactRegistryNginx1176ImageWithDigest = "europe-docker.pkg.dev/gardener-project/releases/3rd/nginx@sha256:b2d89d0a210398b4d1120b3e3a7672c16a4ba09c2c4a0395f18b9f7999b768f2"
-	// RegistryK8sNginx1154ImageWithDigest corresponds to the registry.k8s.io/e2e-test-images/nginx:1.15-4 image.
-	RegistryK8sNginx1154ImageWithDigest = "registry.k8s.io/e2e-test-images/nginx@sha256:db048754ae68ae337d8fa96494c96d2a1204c3320f5dcf7e8e71085adec85da6"
-	// PublicEcrAwsNginx1199ImageWithDigest corresponds to the public.ecr.aws/nginx/nginx:1.19.9 image.
-	PublicEcrAwsNginx1199ImageWithDigest = "public.ecr.aws/nginx/nginx@sha256:f248a8862fa9b21badbe043518f52f143946e2dc705300e8534f8ca624a291b2"
+	// DockerNginx1230Image is the docker.io nginx:1.23.0 image.
+	DockerNginx1230Image = "docker.io/library/nginx:1.23.0"
+	// DockerNginx1240Image is the docker.io nginx:1.24.0 image.
+	DockerNginx1240Image = "docker.io/library/nginx:1.24.0"
+	// DockerNginx1250Image is the docker.io nginx:1.25.0 image.
+	DockerNginx1250Image = "docker.io/library/nginx:1.25.0"
+
+	// ArtifactRegistryNginx1176Image is the europe-docker.pkg.dev/gardener-project/releases/3rd/nginx:1.17.6 image (copy of docker.io/library/nginx:1.17.6).
+	ArtifactRegistryNginx1176Image = "europe-docker.pkg.dev/gardener-project/releases/3rd/nginx:1.17.6"
+	// RegistryK8sNginx1154Image is the registry.k8s.io/e2e-test-images/nginx:1.15-4 image.
+	RegistryK8sNginx1154Image = "registry.k8s.io/e2e-test-images/nginx:1.15-4"
+	// PublicEcrAwsNginx1199Image is the public.ecr.aws/nginx/nginx:1.19.9 image.
+	PublicEcrAwsNginx1199Image = "public.ecr.aws/nginx/nginx:1.19.9"
+
+	// jqExtractRegistryLocation is a jq command that extracts the source location of the '/var/lib/registry' mount from the container's config.json file.
+	jqExtractRegistryLocation = `jq -j '.mounts[] | select(.destination=="/var/lib/registry") | .source' /run/containerd/io.containerd.runtime.v2.task/k8s.io/%s/config.json`
+	// jqCountManifests is a jq command that counts the number of image manifests in the manifest index.
+	// Ref: https://github.com/opencontainers/image-spec/blob/main/image-index.md#example-image-index.
+	jqCountManifests = `jq -j '.manifests | length' %s/docker/registry/v2/blobs/%s/data`
 )
 
 // AddOrUpdateRegistryCacheExtension adds or updates registry-cache extension with the given caches to the given Shoot.
@@ -222,9 +227,9 @@ func VerifyHostsTOMLFilesDeletedForNode(ctx context.Context, rootPodExecutor fra
 // The verification consists of the following steps:
 //  1. It deploys an nginx Pod with the given image.
 //  2. It waits until the Pod is running.
-//  3. It verifies that the image is present in the registry's scheduler-state.json file.
+//  3. It verifies that the image is present in the registry's volume.
 //     This is a verification that the image pull happened via the registry cache (and the containerd didn't fall back to the upstream).
-func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient kubernetes.Interface, upstream, nginxImageWithDigest string) {
+func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient kubernetes.Interface, nginxImage string) {
 	By("Create nginx Pod")
 	ctx, cancel := context.WithTimeout(parentCtx, 5*time.Minute)
 	defer cancel()
@@ -237,7 +242,7 @@ func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient
 			Containers: []corev1.Container{
 				{
 					Name:  "nginx",
-					Image: nginxImageWithDigest,
+					Image: nginxImage,
 				},
 			},
 		},
@@ -251,35 +256,54 @@ func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient
 	ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute)
 	defer cancel()
 
+	upstream, path, tag := splitImage(nginxImage)
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"upstream-host": strings.Replace(upstream, ":", "-", 1)}))
-	EventuallyWithOffset(1, ctx, func() (err error) {
-		reader, err := framework.PodExecByLabel(ctx, selector, "registry-cache", "cat /var/lib/registry/scheduler-state.json", metav1.NamespaceSystem, shootClient)
+	EventuallyWithOffset(1, ctx, func() error {
+		registryPod, err := framework.GetFirstRunningPodWithLabels(ctx, selector, metav1.NamespaceSystem, shootClient)
 		if err != nil {
-			return fmt.Errorf("failed to cat registry's scheduler-state.json file: %+w", err)
+			return fmt.Errorf("failed to get a running registry Pod: %w", err)
 		}
 
-		schedulerStateFileContent, err := io.ReadAll(reader)
+		rootPodExecutor := framework.NewRootPodExecutor(log, shootClient, &registryPod.Spec.NodeName, metav1.NamespaceSystem)
+		defer func(ctx context.Context, rootPodExecutor framework.RootPodExecutor) {
+			_ = rootPodExecutor.Clean(ctx)
+		}(ctx, rootPodExecutor)
+
+		containerID := strings.TrimPrefix(registryPod.Status.ContainerStatuses[0].ContainerID, "containerd://")
+		registryRootPath, err := rootPodExecutor.Execute(ctx, fmt.Sprintf(jqExtractRegistryLocation, containerID))
 		if err != nil {
-			return fmt.Errorf("failed to read from reader: %+w", err)
+			return fmt.Errorf("failed to extract the source localtion of the '/var/lib/registry' mount from the container's config.json file: %w", err)
 		}
 
-		schedulerStateMap := map[string]interface{}{}
-		if err := json.Unmarshal(schedulerStateFileContent, &schedulerStateMap); err != nil {
-			return fmt.Errorf("failed to unmarshal registry's scheduler-state.json file: %+w", err)
+		imageDigest, err := rootPodExecutor.Execute(ctx, fmt.Sprintf("cat %s/docker/registry/v2/repositories/%s/_manifests/tags/%s/current/link", string(registryRootPath), path, tag))
+		if err != nil {
+			return fmt.Errorf("failed to get the %s image digest: %w", nginxImage, err)
 		}
+		imageSha256Value := strings.TrimPrefix(string(imageDigest), "sha256:")
+		imageIndexPath := fmt.Sprintf("sha256/%s/%s", imageSha256Value[:2], imageSha256Value)
 
-		expectedImage := strings.TrimPrefix(nginxImageWithDigest, upstream+"/")
-		if _, ok := schedulerStateMap[expectedImage]; !ok {
-			prettyFileContent, _ := json.MarshalIndent(schedulerStateMap, "", "  ")
-			return fmt.Errorf("failed to find key (image) '%s' in map (registry's scheduler-state.json file) %v", expectedImage, string(prettyFileContent))
+		_, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(jqCountManifests, string(registryRootPath), imageIndexPath))
+		if err != nil {
+			return fmt.Errorf("failed to get the %s image index manifests count: %w", nginxImage, err)
 		}
 
 		return nil
-	}).WithPolling(10*time.Second).Should(Succeed(), "Expected to successfully find the nginx image in the registry's scheduler-state.json file")
+	}).WithPolling(10*time.Second).Should(Succeed(), "Expected to successfully find the nginx image in the registry's volume")
 
 	By("Delete nginx Pod")
 	timeout := 5 * time.Minute
 	ctx, cancel = context.WithTimeout(parentCtx, timeout)
 	defer cancel()
 	ExpectWithOffset(1, framework.DeleteAndWaitForResource(ctx, shootClient, pod, timeout)).To(Succeed())
 }
+
+// splitImage splits the image to <upstream>/<path>:<tag>
+func splitImage(image string) (upstream, path, tag string) {
+	index := strings.Index(image, "/")
+	upstream = image[:index]
+	path = image[index+1:]
+	index = strings.LastIndex(path, ":")
+	tag = path[index+1:]
+	path = path[:index]
+	return
+}

test/e2e/cache/create_enable_add_remove_disable_delete.go

@@ -49,7 +49,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() {
 		common.WaitUntilRegistryCacheConfigurationsAreApplied(ctx, f.Logger, f.ShootFramework.ShootClient)
 
 		By("[docker.io] Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, "docker.io", common.DockerNginx1230ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.DockerNginx1230Image)
 
 		By("Add the public.ecr.aws upstream to the registry-cache extension")
 		ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)
@@ -70,7 +70,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() {
 		common.WaitUntilRegistryCacheConfigurationsAreApplied(ctx, f.Logger, f.ShootFramework.ShootClient)
 
 		By("[public.ecr.aws] Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, "public.ecr.aws", common.PublicEcrAwsNginx1199ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.PublicEcrAwsNginx1199Image)
 
 		By("Remove the public.ecr.aws upstream from the registry-cache extension")
 		ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)

test/e2e/cache/create_enabled_delete_shoot_system_components.go

@@ -46,10 +46,10 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() {
 		common.WaitUntilRegistryCacheConfigurationsAreApplied(ctx, f.Logger, f.ShootFramework.ShootClient)
 
 		By("[europe-docker.pkg.dev] Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, "europe-docker.pkg.dev", common.ArtifactRegistryNginx1176ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.ArtifactRegistryNginx1176Image)
 
 		By("[registry.k8s.io] Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, "registry.k8s.io", common.RegistryK8sNginx1154ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.RegistryK8sNginx1154Image)
 
 		By("Delete Shoot")
 		ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute)

test/e2e/cache/create_enabled_force_delete.go

@@ -41,7 +41,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() {
 		common.WaitUntilRegistryCacheConfigurationsAreApplied(ctx, f.Logger, f.ShootFramework.ShootClient)
 
 		By("Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, "docker.io", common.DockerNginx1230ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.DockerNginx1230Image)
 
 		By("Force Delete Shoot")
 		ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)

test/e2e/cache/create_enabled_hibernate_reconcile_delete.go

@@ -43,7 +43,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() {
 		common.WaitUntilRegistryCacheConfigurationsAreApplied(ctx, f.Logger, f.ShootFramework.ShootClient)
 
 		By("Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, "docker.io", common.DockerNginx1230ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.DockerNginx1230Image)
 
 		By("Hibernate Shoot")
 		ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)

test/testmachinery/shoot/enable_disable_test.go

@@ -50,7 +50,7 @@ var _ = Describe("Shoot registry cache testing", func() {
 		common.WaitUntilRegistryCacheConfigurationsAreApplied(ctx, f.Logger, f.ShootClient)
 
 		By("Verify registry-cache works")
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, "docker.io", common.DockerNginx1230ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.DockerNginx1230Image)
 
 		By("Disable the registry-cache extension")
 		ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)

test/testmachinery/shoot/enable_hibernate_reconcile_wakeup_disable_test.go

@@ -54,7 +54,7 @@ var _ = Describe("Shoot registry cache testing", func() {
 		By("Verify registry-cache works")
 		// We are using nginx:1.24.0 as nginx:1.23.0 is already used by the "should enable and disable the registry-cache extension" test.
 		// Hence, nginx:1.23.0 will be present in the Node.
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, "docker.io", common.DockerNginx1240ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.DockerNginx1240Image)
 
 		By("Hibernate Shoot")
 		ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute)
@@ -83,7 +83,7 @@ var _ = Describe("Shoot registry cache testing", func() {
 
 		By("Verify registry-cache works after wake up")
 		// We are using nginx:1.25.0 as nginx:1.24.0 is already used above and already present in the Node and in the registry cache.
-		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, "docker.io", common.DockerNginx1250ImageWithDigest)
+		common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.DockerNginx1250Image)
 	}, hibernationTestTimeout, framework.WithCAfterTest(func(ctx context.Context) {
 		if v1beta1helper.HibernationIsEnabled(f.Shoot) {
 			By("Wake up Shoot")