libvirt: e2e test for attestation secret retrieve for sample tee

Fixes: #1825

Signed-off-by: Qi Feng Huo <huoqif@cn.ibm.com>

src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml

@@ -23,6 +23,7 @@ configMapGenerator:
   - LIBVIRT_NET="default" # set
   - LIBVIRT_POOL="default" # set
   - DISABLECVM="true" # set as false to enable confidential VM
+  - AA_KBC_PARAMS="" #set KBC params for podvm
   #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv
   #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2

src/cloud-api-adaptor/libvirt/README.md

@@ -212,6 +212,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2
 * ``TEST_PODVM_IMAGE`` - image to be used for this testing
 * ``CLOUD_PROVIDER`` - which cloud provider should be used
 * ``TEST_E2E_TIMEOUT`` - test timeout
+* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow
 * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh))
 
 # Delete Confidential Containers and cloud-api-adaptor from the cluster

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -6,6 +6,7 @@
 package e2e
 
 import (
+	"os"
 	"testing"
 
 	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt"
@@ -96,3 +97,12 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) {
 	assert := LibvirtAssert{}
 	DoTestPodsMTLSCommunication(t, testEnv, assert)
 }
+
+func TestLibvirtKbsKeyRelease(t *testing.T) {
+	if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" {
+		t.Skip("Skipping kbs related test as kbs is not deployed")
+	}
+	assert := LibvirtAssert{}
+	t.Parallel()
+	DoTestKbsKeyRelease(t, testEnv, assert)
+}

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
+		"network":       {"default", "LIBVIRT_NET"},
+		"storage":       {"default", "LIBVIRT_POOL"},
+		"pause_image":   {"", "PAUSE_IMAGE"},
+		"podvm_volume":  {"", "LIBVIRT_VOL_NAME"},
+		"uri":           {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
+		"vxlan_port":    {"", "VXLAN_PORT"},
+		"AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"},
 	}
 
 	for k, v := range mapProps {

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -81,7 +81,7 @@ type InstallOverlay interface {
 const PodWaitTimeout = time.Second * 30
 
 // trustee repo related base path
-const TRUSTEE_REPO_PATH = "../trustee"
+const TRUSTEE_REPO_PATH = "trustee"
 
 func saveToFile(filename string, content []byte) error {
 	// Save contents to file
@@ -134,16 +134,23 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		}
 		defer keyOutputFile.Close()
 
-		_, privateKey, err := ed25519.GenerateKey(rand.Reader)
+		pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
 			err = fmt.Errorf("generating Ed25519 key pair: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
+		b, err := x509.MarshalPKCS8PrivateKey(privateKey)
+		if err != nil {
+			err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err)
+			log.Errorf("%v", err)
+			return nil, err
+		}
+
 		privateKeyPEM := pem.EncodeToMemory(&pem.Block{
 			Type:  "PRIVATE KEY",
-			Bytes: privateKey,
+			Bytes: b,
 		})
 
 		// Save private key to file
@@ -154,17 +161,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 			return nil, err
 		}
 
-		publicKey := privateKey.Public().(ed25519.PublicKey)
-		publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey)
+		b, err = x509.MarshalPKIXPublicKey(pubKey)
 		if err != nil {
-			err = fmt.Errorf("generating Ed25519 public key: %w\n", err)
+			err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
 		publicKeyPEM := pem.EncodeToMemory(&pem.Block{
 			Type:  "PUBLIC KEY",
-			Bytes: publicKeyX509,
+			Bytes: b,
 		})
 
 		// Save public key to file
@@ -335,6 +341,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 
 	resources := client.Resources(namespace)
 
+	kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}}
+	fmt.Printf("Wait for the %s deployment be available\n", deploymentName)
+	if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue),
+		wait.WithTimeout(time.Minute*2)); err != nil {
+		return "", err
+	}
+
 	services := &corev1.ServiceList{}
 	if err := resources.List(context.TODO(), services); err != nil {
 		return "", err
@@ -530,17 +543,7 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 	}
 
 	for ds, timeout := range daemonSetList {
-		// Wait for the daemonset to have at least one pod running then wait for each pod
-		// be ready.
-
-		fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName())
-		if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool {
-			ds = object.(*appsv1.DaemonSet)
-
-			return ds.Status.CurrentNumberScheduled > 0
-		}), wait.WithTimeout(time.Minute*5)); err != nil {
-			return err
-		}
+		// Wait for the daemonset to have at least one pod ready
 		pods, err := GetDaemonSetOwnedPods(ctx, cfg, ds)
 		if err != nil {
 			return err