From 529cde5f2b15851c05df001733029d605ef0a0f1 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Sun, 28 Apr 2024 15:36:55 +0800 Subject: [PATCH] libvirt: e2e test for attestation secret retrieve for sample tee Fixes: #1825 Signed-off-by: Qi Feng Huo --- .../overlays/libvirt/kustomization.yaml | 1 + src/cloud-api-adaptor/libvirt/README.md | 1 + .../test/e2e/common_suite.go | 23 +++++++++++++++++++ .../test/e2e/libvirt_test.go | 9 ++++++++ .../provisioner/libvirt/provision_common.go | 13 ++++++----- .../test/provisioner/provision.go | 19 +++++++-------- 6 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 180ae90c75..49f4cc023c 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -23,6 +23,7 @@ configMapGenerator: - LIBVIRT_NET="default" # set - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM + - AA_KBC_PARAMS="" #set KBC params for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/libvirt/README.md b/src/cloud-api-adaptor/libvirt/README.md index 9076333fa1..916a4c2735 100644 --- a/src/cloud-api-adaptor/libvirt/README.md +++ b/src/cloud-api-adaptor/libvirt/README.md @@ -212,6 +212,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2 * ``TEST_PODVM_IMAGE`` - image to be used for this testing * ``CLOUD_PROVIDER`` - which cloud provider should be used * ``TEST_E2E_TIMEOUT`` - test timeout +* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh)) # Delete Confidential Containers and cloud-api-adaptor from the cluster diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 616b7a7e5a..2b9143d9dc 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -572,3 +572,26 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs NewTestCase(t, e, "TestPodsMTLSCommunication", assert, "Pods communication with mTLS").WithPod(serverPod).WithExtraPods(extraPods).WithConfigMap(configMap).WithService(nginxSvc).WithSecret(serverSecret).WithExtraSecrets(extraSecrets).Run() } + +func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { + + log.Info("Do test kbs key release") + pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget") + testCommands := []TestCommand{ + { + Command: []string{"wget", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, + ContainerName: pod.Spec.Containers[0].Name, + TestCommandStdoutFn: func(stdout bytes.Buffer) bool { + if strings.Contains(stdout.String(), "This is my cluster name") { + log.Infof("Success to get key.bin %s", stdout.String()) + return true + } else { + log.Errorf("Failed to access key.bin: %s", stdout.String()) + return false + } + }, + }, + } + + NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run() +} diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 3be18c9daf..2e7469f030 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -6,6 +6,7 @@ package e2e import ( + "os" "testing" _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt" @@ -96,3 +97,11 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) { assert := LibvirtAssert{} DoTestPodsMTLSCommunication(t, testEnv, assert) } + +func TestKbsKeyRelease(t *testing.T) { + if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + t.Parallel() + DoTestKbsKeyRelease(t, testEnv, assert) +} diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index c5f65cfc12..22bc0d796b 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 72cdbb41bf..374285a830 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -332,6 +332,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf resources := client.Resources(namespace) + kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}} + fmt.Printf("Wait for the %s deployment be available\n", deploymentName) + if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*2)); err != nil { + return "", err + } + services := &corev1.ServiceList{} if err := resources.List(context.TODO(), services); err != nil { return "", err @@ -527,17 +534,7 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props } for ds, timeout := range daemonSetList { - // Wait for the daemonset to have at least one pod running then wait for each pod - // be ready. - - fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName()) - if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool { - ds = object.(*appsv1.DaemonSet) - - return ds.Status.CurrentNumberScheduled > 0 - }), wait.WithTimeout(time.Minute*5)); err != nil { - return err - } + // Wait for the daemonset to have at least one pod ready pods, err := GetDaemonSetOwnedPods(ctx, cfg, ds) if err != nil { return err