e2e: use busybox to reduce unexpected e2e-test fail

- sometime nginx container not work well in te2e test, make the test cases fail - use busybox to reduce this unexpected fails - add a new nginx deployment test to tract the nginx image issue fixes for confidential-containers#1450 Signed-off-by: Da Li Liu <liudali@cn.ibm.com>
liudalibj · Dec 20, 2023 · b0c3f24 · b0c3f24
1 parent c70971e
commit b0c3f24
Show file tree

Hide file tree

Showing 7 changed files with 293 additions and 89 deletions.
diff --git a/test/e2e/README.md b/test/e2e/README.md
@@ -161,3 +161,23 @@ TEST_PROVISION=no \
 TEST_INSTALL_CAA=no \
 TEST_PROVISION_FILE="${PWD}/skip-provisioning.properties" \
 ```
+## Failed to run nginx container in peer pod
+The 'nginx:latest' is used some of e2e test cases before, in the daily e2e-test report we found that this image make the test result not stable. The nginx container is CrashLoopBackOff status and the pod log looks like:
+```
+/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
+/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
+/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
+10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
+dpkg-query: no packages found matching nginx
+10-listen-on-ipv6-by-default.sh: info: /etc/nginx/conf.d/default.conf differs from the packaged version
+/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
+/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
+/docker-entrypoint.sh: Ignoring /docker-entrypoint.d/.wh..wh..opq
+/docker-entrypoint.sh: Configuration complete; ready for start up
+2023/09/19 13:56:25 [emerg] 1#1: getpwnam("nginx") failed in /etc/nginx/nginx.conf:2
+nginx: [emerg] getpwnam("nginx") failed in /etc/nginx/nginx.conf:2
+```
+The issue should be fixed in https://github.com/confidential-containers/guest-components/issues/403 but at the moment, a fix cannot be applied to kata, b/c of an issue w/ guest-components.
+
+We discussed this on the community call(2023/12/06) and the suggestion is that whilst we don't want to hide and lose the nginx issue, having a range of tests that occasionally fail isn't helpful either, so the propose that we swap the defaultimages for tests to busybox.
+The new added nginx specific tesst `TestIBMCloudCreateNginxDeployment` and `TestLibvirtCreateNginxDeployment`.
diff --git a/test/e2e/assessment_runner_test.go b/test/e2e/assessment_runner_test.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -189,45 +190,33 @@ func (tc *testCase) run() {
 				if err = wait.For(conditions.New(client.Resources()).PodPhaseMatch(tc.pod, tc.podState), wait.WithTimeout(WAIT_POD_RUNNING_TIMEOUT)); err != nil {
 					t.Fatal(err)
 				}
-				if tc.podState == v1.PodRunning || tc.pod.Spec.Containers[0].ReadinessProbe != nil {
-					clientset, err := kubernetes.NewForConfig(client.RESTConfig())
-					if err != nil {
-						t.Fatal(err)
-					}
-					pod, err := clientset.CoreV1().Pods(tc.pod.Namespace).Get(ctx, tc.pod.Name, metav1.GetOptions{})
-					if err != nil {
+				if tc.podState == v1.PodRunning || len(tc.testCommands) > 0 {
+					t.Logf("Waiting for containers in pod: %v are ready", tc.pod.Name)
+					if err = wait.For(conditions.New(client.Resources()).ContainersReady(tc.pod), wait.WithTimeout(WAIT_POD_RUNNING_TIMEOUT)); err != nil {
+						//Added logs for debugging nightly tests
+						clientset, err := kubernetes.NewForConfig(client.RESTConfig())
+						if err != nil {
+							t.Fatal(err)
+						}
+						pod, err := clientset.CoreV1().Pods(tc.pod.Namespace).Get(ctx, tc.pod.Name, metav1.GetOptions{})
+						if err != nil {
+							t.Fatal(err)
+						}
+						t.Logf("Expected Pod State: %v", tc.podState)
+						yamlData, err := yaml.Marshal(pod.Status)
+						if err != nil {
+							fmt.Println("Error marshaling pod.Status to YAML: ", err.Error())
+						} else {
+							t.Logf("Current Pod State: %v", string(yamlData))
+						}
+						if pod.Status.Phase == v1.PodRunning {
+							fmt.Printf("Log of the pod %.v \n===================\n", pod.Name)
+							podLogString, _ := getPodLog(ctx, client, *pod)
+							fmt.Println(podLogString)
+							fmt.Printf("===================\n")
+						}
 						t.Fatal(err)
 					}
-					//Added logs for debugging nightly tests
-					t.Logf("Expected Pod State: %v", tc.podState)
-					t.Logf("Current Pod State: %v", pod.Status.Phase)
-					//Getting Readiness probe of a container
-					for i, condition := range pod.Status.Conditions {
-						fmt.Printf("===================\n")
-						fmt.Printf("Checking Conditons - %v....\n", i+1)
-						fmt.Printf("===================\n")
-						fmt.Printf("*.Condition Type: %v\n", condition.Type)
-						fmt.Printf("*.Condition Status: %v\n", condition.Status)
-						fmt.Printf("*.Condition Last Probe Time: %v\n", condition.LastProbeTime)
-						fmt.Printf("*.Condition Last Transition Time: %v\n", condition.LastTransitionTime)
-						fmt.Printf("*.Condition Last Message: %v\n", condition.Message)
-						fmt.Printf("*.Condition Last Reason: %v\n", condition.Reason)
-					}
-
-					readinessProbe := pod.Spec.Containers[0].ReadinessProbe
-					if readinessProbe != nil {
-						fmt.Printf("===================\n")
-						fmt.Printf("Checking Readiness Probe....\n")
-						fmt.Printf("===================\n")
-						fmt.Printf("*.Initial Delay Seconds: %v\n", readinessProbe.InitialDelaySeconds)
-						fmt.Printf("*.Timeout Seconds: %v\n", readinessProbe.TimeoutSeconds)
-						fmt.Printf("*.Success Threshold: %v\n", readinessProbe.SuccessThreshold)
-						fmt.Printf("*.Failure Threshold: %v\n", readinessProbe.FailureThreshold)
-						fmt.Printf("*.Period Seconds: %v\n", readinessProbe.PeriodSeconds)
-						fmt.Printf("*.Probe Handler: %v\n", readinessProbe.ProbeHandler)
-						fmt.Printf("*.Probe Handler Port: %v\n", readinessProbe.ProbeHandler.HTTPGet.Port)
-						fmt.Printf("===================\n")
-					}
 				}
 			}
 

diff --git a/test/e2e/common.go b/test/e2e/common.go
@@ -8,9 +8,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest"
+
 type podOption func(*corev1.Pod)
 
 func withRestartPolicy(restartPolicy corev1.RestartPolicy) podOption {
@@ -19,23 +20,6 @@ func withRestartPolicy(restartPolicy corev1.RestartPolicy) podOption {
 	}
 }
 
-// Optional method to add ContainerPort and ReadinessProbe to listen Port 80
-func withContainerPort(port int32) podOption {
-	return func(p *corev1.Pod) {
-		p.Spec.Containers[0].Ports = []corev1.ContainerPort{{ContainerPort: port}}
-		p.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
-			ProbeHandler: corev1.ProbeHandler{
-				HTTPGet: &corev1.HTTPGetAction{
-					Path: "/",
-					Port: intstr.FromInt(int(port)),
-				},
-			},
-			InitialDelaySeconds: 10,
-			PeriodSeconds:       5,
-		}
-	}
-}
-
 func withCommand(command []string) podOption {
 	return func(p *corev1.Pod) {
 		p.Spec.Containers[0].Command = command
@@ -107,28 +91,24 @@ func newPod(namespace string, podName string, containerName string, imageName st
 	return pod
 }
 
-func newNginxPod(namespace string) *corev1.Pod {
-	return newPod(namespace, "nginx", "nginx", "nginx", withRestartPolicy(corev1.RestartPolicyNever))
-}
-
-func newNginxPodWithName(namespace string, podName string) *corev1.Pod {
-	return newPod(namespace, podName, "nginx", "nginx", withRestartPolicy(corev1.RestartPolicyNever))
+func newBusyboxPod(namespace string) *corev1.Pod {
+	return newPod(namespace, "busybox-pod", "busybox", BUSYBOX_IMAGE, withCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
 }
 
-func newBusyboxPod(namespace string) *corev1.Pod {
-	return newPod(namespace, "busybox-pod", "busybox", "quay.io/prometheus/busybox:latest", withCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
+func newBusyboxPodWithName(namespace, podName string) *corev1.Pod {
+	return newPod(namespace, podName, "busybox", BUSYBOX_IMAGE, withCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
 }
 
 // newConfigMap returns a new config map object.
-func newConfigMap(namespace string, name string, configMapData map[string]string) *corev1.ConfigMap {
+func newConfigMap(namespace, name string, configMapData map[string]string) *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
 		Data:       configMapData,
 	}
 }
 
 // newSecret returns a new secret object.
-func newSecret(namespace string, name string, data map[string][]byte, secretType corev1.SecretType) *corev1.Secret {
+func newSecret(namespace, name string, data map[string][]byte, secretType corev1.SecretType) *corev1.Secret {
 	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
 		Data:       data,
@@ -137,7 +117,7 @@ func newSecret(namespace string, name string, data map[string][]byte, secretType
 }
 
 // newJob returns a new job
-func newJob(namespace string, name string) *batchv1.Job {
+func newJob(namespace, name string) *batchv1.Job {
 	runtimeClassName := "kata-remote"
 	// Comment out adding runtime-handler until nydus-snapshotter is stable
 	// annotationData := map[string]string{

diff --git a/test/e2e/common_suite_test.go b/test/e2e/common_suite_test.go
@@ -14,15 +14,14 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	envconf "sigs.k8s.io/e2e-framework/pkg/envconf"
 )
 
 // doTestCreateSimplePod tests a simple peer-pod can be created.
 func doTestCreateSimplePod(t *testing.T, assert CloudAssert) {
 	namespace := envconf.RandomName("default", 7)
-	pod := newNginxPod(namespace)
+	pod := newBusyboxPod(namespace)
 	newTestCase(t, "SimplePeerPod", assert, "PodVM is created").withPod(pod).run()
 }
 
@@ -31,29 +30,29 @@ func doTestCreateSimplePodWithNydusAnnotation(t *testing.T, assert CloudAssert)
 	annotationData := map[string]string{
 		"io.containerd.cri.runtime-handler": "kata-remote",
 	}
-	pod := newPod(namespace, "alpine", "alpine", "alpine", withRestartPolicy(corev1.RestartPolicyNever), withAnnotations(annotationData))
+	pod := newPod(namespace, "alpine", "alpine", "alpine", withRestartPolicy(v1.RestartPolicyNever), withAnnotations(annotationData))
 	newTestCase(t, "SimplePeerPod", assert, "PodVM is created").withPod(pod).withNydusSnapshotter().run()
 }
 
 func doTestDeleteSimplePod(t *testing.T, assert CloudAssert) {
 	namespace := envconf.RandomName("default", 7)
-	pod := newNginxPodWithName(namespace, "deletion-test")
+	pod := newBusyboxPodWithName(namespace, "deletion-test")
 	duration := 1 * time.Minute
 	newTestCase(t, "DeletePod", assert, "Deletion complete").withPod(pod).withDeleteAssertion(&duration).run()
 }
 
 func doTestCreatePodWithConfigMap(t *testing.T, assert CloudAssert) {
 	namespace := envconf.RandomName("default", 7)
-	podName := "nginx-configmap-pod"
-	containerName := "nginx-configmap-container"
-	imageName := "nginx:latest"
-	configMapName := "nginx-configmap"
+	podName := "busybox-configmap-pod"
+	containerName := "busybox-configmap-container"
+	imageName := BUSYBOX_IMAGE
+	configMapName := "busybox-configmap"
 	configMapFileName := "example.txt"
 	podKubeConfigmapDir := "/etc/config/"
 	configMapPath := podKubeConfigmapDir + configMapFileName
 	configMapContents := "Hello, world"
 	configMapData := map[string]string{configMapFileName: configMapContents}
-	pod := newPod(namespace, podName, containerName, imageName, withConfigMapBinding(podKubeConfigmapDir, configMapName), withContainerPort(80))
+	pod := newPod(namespace, podName, containerName, imageName, withConfigMapBinding(podKubeConfigmapDir, configMapName), withCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
 	configMap := newConfigMap(namespace, configMapName, configMapData)
 	testCommands := []testCommand{
 		{
@@ -77,10 +76,10 @@ func doTestCreatePodWithConfigMap(t *testing.T, assert CloudAssert) {
 func doTestCreatePodWithSecret(t *testing.T, assert CloudAssert) {
 	//doTestCreatePod(t, assert, "Secret is created and contains data", pod)
 	namespace := envconf.RandomName("default", 7)
-	podName := "nginx-secret-pod"
-	containerName := "nginx-secret-container"
-	imageName := "nginx:latest"
-	secretName := "nginx-secret"
+	podName := "busybox-secret-pod"
+	containerName := "busybox-secret-container"
+	imageName := BUSYBOX_IMAGE
+	secretName := "busybox-secret"
 	podKubeSecretsDir := "/etc/secret/"
 	usernameFileName := "username"
 	username := "admin"
@@ -89,7 +88,7 @@ func doTestCreatePodWithSecret(t *testing.T, assert CloudAssert) {
 	password := "password"
 	passwordPath := podKubeSecretsDir + passwordFileName
 	secretData := map[string][]byte{passwordFileName: []byte(password), usernameFileName: []byte(username)}
-	pod := newPod(namespace, podName, containerName, imageName, withSecretBinding(podKubeSecretsDir, secretName), withContainerPort(80))
+	pod := newPod(namespace, podName, containerName, imageName, withSecretBinding(podKubeSecretsDir, secretName), withCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
 	secret := newSecret(namespace, secretName, secretData, v1.SecretTypeOpaque)
 
 	testCommands := []testCommand{
@@ -168,7 +167,7 @@ func doTestCreatePeerPodAndCheckUserLogs(t *testing.T, assert CloudAssert) {
 // doTestCreateConfidentialPod verify a confidential peer-pod can be created.
 func doTestCreateConfidentialPod(t *testing.T, assert CloudAssert, testCommands []testCommand) {
 	namespace := envconf.RandomName("default", 7)
-	pod := newNginxPodWithName(namespace, "confidential-pod-nginx")
+	pod := newBusyboxPodWithName(namespace, "confidential-pod-busybox")
 	for i := 0; i < len(testCommands); i++ {
 		testCommands[i].containerName = pod.Spec.Containers[0].Name
 	}
@@ -197,8 +196,8 @@ func doTestCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t *testing.T, asser
 func doTestCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t *testing.T, assert CloudAssert) {
 	namespace := envconf.RandomName("default", 7)
 	podName := "env-variable-in-config"
-	imageName := "nginx:latest"
-	pod := newPod(namespace, podName, podName, imageName, withRestartPolicy(v1.RestartPolicyOnFailure), withEnvironmentalVariables([]v1.EnvVar{{Name: "ISPRODUCTION", Value: "true"}}), withCommand([]string{"/bin/sh", "-c", "env"}), withContainerPort(80))
+	imageName := BUSYBOX_IMAGE
+	pod := newPod(namespace, podName, podName, imageName, withRestartPolicy(v1.RestartPolicyOnFailure), withEnvironmentalVariables([]v1.EnvVar{{Name: "ISPRODUCTION", Value: "true"}}), withCommand([]string{"/bin/sh", "-c", "env"}))
 	expectedPodLogString := "ISPRODUCTION=true"
 	newTestCase(t, "EnvVariablePeerPodWithDeploymentOnly", assert, "Peer pod with environmental variables has been created").withPod(pod).withExpectedPodLogString(expectedPodLogString).withCustomPodState(v1.PodSucceeded).run()
 }
@@ -298,7 +297,7 @@ func doTestPodVMwithNoAnnotations(t *testing.T, assert CloudAssert, expectedType
 	namespace := envconf.RandomName("default", 7)
 	podName := "no-annotations"
 	containerName := "busybox"
-	imageName := "busybox:latest"
+	imageName := BUSYBOX_IMAGE
 	pod := newPod(namespace, podName, containerName, imageName, withCommand([]string{"/bin/sh", "-c", "sleep 3600"}))
 	testInstanceTypes := instanceValidatorFunctions{
 		testSuccessfn: func(instance string) bool {
@@ -319,7 +318,7 @@ func doTestPodVMwithAnnotationsInstanceType(t *testing.T, assert CloudAssert, ex
 	namespace := envconf.RandomName("default", 7)
 	podName := "annotations-instance-type"
 	containerName := "busybox"
-	imageName := "busybox:latest"
+	imageName := BUSYBOX_IMAGE
 	annotationData := map[string]string{
 		"io.katacontainers.config.hypervisor.machine_type": expectedType,
 	}
@@ -344,7 +343,7 @@ func doTestPodVMwithAnnotationsCPUMemory(t *testing.T, assert CloudAssert, expec
 	namespace := envconf.RandomName("default", 7)
 	podName := "annotations-cpu-mem"
 	containerName := "busybox"
-	imageName := "busybox:latest"
+	imageName := BUSYBOX_IMAGE
 	annotationData := map[string]string{
 		"io.katacontainers.config.hypervisor.default_vcpus":  "2",
 		"io.katacontainers.config.hypervisor.default_memory": "12288",
@@ -370,7 +369,7 @@ func doTestPodVMwithAnnotationsInvalidInstanceType(t *testing.T, assert CloudAss
 	namespace := envconf.RandomName("default", 7)
 	podName := "annotations-invalid-instance-type"
 	containerName := "busybox"
-	imageName := "busybox:latest"
+	imageName := BUSYBOX_IMAGE
 	expectedErrorMessage := `requested instance type ("` + expectedType + `") is not part of supported instance types list`
 	annotationData := map[string]string{
 		"io.katacontainers.config.hypervisor.machine_type": expectedType,
@@ -395,7 +394,7 @@ func doTestPodVMwithAnnotationsLargerMemory(t *testing.T, assert CloudAssert) {
 	namespace := envconf.RandomName("default", 7)
 	podName := "annotations-too-big-mem"
 	containerName := "busybox"
-	imageName := "busybox:latest"
+	imageName := BUSYBOX_IMAGE
 	expectedErrorMessage := "failed to get instance type based on vCPU and memory annotations: no instance type found for the given vcpus (2) and memory (18432)"
 	annotationData := map[string]string{
 		"io.katacontainers.config.hypervisor.default_vcpus":  "2",
@@ -421,7 +420,7 @@ func doTestPodVMwithAnnotationsLargerCPU(t *testing.T, assert CloudAssert) {
 	namespace := envconf.RandomName("default", 7)
 	podName := "annotations-too-big-cpu"
 	containerName := "busybox"
-	imageName := "busybox:latest"
+	imageName := BUSYBOX_IMAGE
 	expectedErrorMessage := []string{
 		"no instance type found for the given vcpus (3) and memory (12288)",
 		"Number of cpus 3 specified in annotation default_vcpus is greater than the number of CPUs 2 on the system",

diff --git a/test/e2e/ibmcloud_test.go b/test/e2e/ibmcloud_test.go
@@ -495,6 +495,15 @@ func TestPodVMwithAnnotationsLargerCPU(t *testing.T) {
 	doTestPodVMwithAnnotationsLargerCPU(t, assert)
 }
 
+func TestIBMCloudCreateNginxDeployment(t *testing.T) {
+	// TODO: https://github.com/confidential-containers/guest-components/issues/403
+	t.Skip("Skipping Test until https://github.com/confidential-containers/guest-components/issues/403 be applied to kata")
+	// assert := IBMCloudAssert{
+	// 	vpc: pv.IBMCloudProps.VPC,
+	// }
+	// doTestNginxDeployement(t, assert)
+}
+
 // IBMCloudAssert implements the CloudAssert interface for ibmcloud.
 type IBMCloudAssert struct {
 	vpc *vpcv1.VpcV1

diff --git a/test/e2e/libvirt_test.go b/test/e2e/libvirt_test.go
@@ -67,6 +67,13 @@ func TestLibvirtCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t *te
 	doTestCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t, assert)
 }
 
+func TestLibvirtCreateNginxDeployment(t *testing.T) {
+	// TODO: https://github.com/confidential-containers/guest-components/issues/403
+	t.Skip("Skipping Test until https://github.com/confidential-containers/guest-components/issues/403 be applied to kata")
+	// assert := LibvirtAssert{}
+	// doTestNginxDeployement(t, assert)
+}
+
 /*
 Failing due to issues will pulling image (ErrImagePull)
 func TestLibvirtCreatePeerPodWithLargeImage(t *testing.T) {