libvirt: e2e test for attestation secret retrieve for sample tee

Fixes: #1825

Signed-off-by: Qi Feng Huo <huoqif@cn.ibm.com>

src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml

@@ -23,6 +23,7 @@ configMapGenerator:
   - LIBVIRT_NET="default" # set
   - LIBVIRT_POOL="default" # set
   - DISABLECVM="true" # set as false to enable confidential VM
+  - AA_KBC_PARAMS="" #set KBC params for podvm
   #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv
   #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2

src/cloud-api-adaptor/libvirt/README.md

@@ -212,6 +212,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2
 * ``TEST_PODVM_IMAGE`` - image to be used for this testing
 * ``CLOUD_PROVIDER`` - which cloud provider should be used
 * ``TEST_E2E_TIMEOUT`` - test timeout
+* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow
 * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh))
 
 # Delete Confidential Containers and cloud-api-adaptor from the cluster

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -572,3 +572,26 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs
 	NewTestCase(t, e, "TestPodsMTLSCommunication", assert, "Pods communication with mTLS").WithPod(serverPod).WithExtraPods(extraPods).WithConfigMap(configMap).WithService(nginxSvc).WithSecret(serverSecret).WithExtraSecrets(extraSecrets).Run()
 
 }
+
+func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) {
+
+	log.Info("Do test kbs key release")
+	pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget")
+	testCommands := []TestCommand{
+		{
+			Command:       []string{"wget", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"},
+			ContainerName: pod.Spec.Containers[0].Name,
+			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
+				if strings.Contains(stdout.String(), "This is my cluster name") {
+					log.Infof("Success to get key.bin %s", stdout.String())
+					return true
+				} else {
+					log.Errorf("Failed to access key.bin: %s", stdout.String())
+					return false
+				}
+			},
+		},
+	}
+
+	NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run()
+}

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -6,6 +6,7 @@
 package e2e
 
 import (
+	"os"
 	"testing"
 
 	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt"
@@ -96,3 +97,11 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) {
 	assert := LibvirtAssert{}
 	DoTestPodsMTLSCommunication(t, testEnv, assert)
 }
+
+func TestKbsKeyRelease(t *testing.T) {
+	if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" {
+		t.Skip("Skipping kbs related test as kbs is not deployed")
+	}
+	t.Parallel()
+	DoTestKbsKeyRelease(t, testEnv, assert)
+}

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
+		"network":         {"default", "LIBVIRT_NET"},
+		"storage":         {"default", "LIBVIRT_POOL"},
+		"pause_image":     {"", "PAUSE_IMAGE"},
+		"podvm_volume":    {"", "LIBVIRT_VOL_NAME"},
+		"uri":             {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
+		"vxlan_port":      {"", "VXLAN_PORT"},
+		"AA_KBC_PARAMS":   {"", "AA_KBC_PARAMS"},
 	}
 
 	for k, v := range mapProps {

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -332,6 +332,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 
 	resources := client.Resources(namespace)
 
+	kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}}
+	fmt.Printf("Wait for the %s deployment be available\n", deploymentName)
+	if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue),
+		wait.WithTimeout(time.Minute*2)); err != nil {
+		return "", err
+	}
+
 	services := &corev1.ServiceList{}
 	if err := resources.List(context.TODO(), services); err != nil {
 		return "", err
@@ -527,17 +534,7 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 	}
 
 	for ds, timeout := range daemonSetList {
-		// Wait for the daemonset to have at least one pod running then wait for each pod
-		// be ready.
-
-		fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName())
-		if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool {
-			ds = object.(*appsv1.DaemonSet)
-
-			return ds.Status.CurrentNumberScheduled > 0
-		}), wait.WithTimeout(time.Minute*5)); err != nil {
-			return err
-		}
+		// Wait for the daemonset to have at least one pod ready
 		pods, err := GetDaemonSetOwnedPods(ctx, cfg, ds)
 		if err != nil {
 			return err