From 8a45203e22b2234f76d2a32056fadfd1032ba628 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Sun, 28 Apr 2024 15:36:55 +0800 Subject: [PATCH] libvirt: e2e test for attestation secret retrieve for sample tee Fixes: #1825 Signed-off-by: Qi Feng Huo --- .../overlays/libvirt/kustomization.yaml | 1 + src/cloud-api-adaptor/libvirt/README.md | 1 + .../test/e2e/libvirt_test.go | 10 +++++ .../provisioner/libvirt/provision_common.go | 13 ++++--- .../test/provisioner/provision.go | 39 ++++++++++--------- 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 180ae90c75..49f4cc023c 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -23,6 +23,7 @@ configMapGenerator: - LIBVIRT_NET="default" # set - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM + - AA_KBC_PARAMS="" #set KBC params for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/libvirt/README.md b/src/cloud-api-adaptor/libvirt/README.md index 9076333fa1..916a4c2735 100644 --- a/src/cloud-api-adaptor/libvirt/README.md +++ b/src/cloud-api-adaptor/libvirt/README.md @@ -212,6 +212,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2 * ``TEST_PODVM_IMAGE`` - image to be used for this testing * ``CLOUD_PROVIDER`` - which cloud provider should be used * ``TEST_E2E_TIMEOUT`` - test timeout +* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh)) # Delete Confidential Containers and cloud-api-adaptor from the cluster diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 3be18c9daf..73be500d6a 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -6,6 +6,7 @@ package e2e import ( + "os" "testing" _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt" @@ -96,3 +97,12 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) { assert := LibvirtAssert{} DoTestPodsMTLSCommunication(t, testEnv, assert) } + +func TestLibvirtKbsKeyRelease(t *testing.T) { + if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + assert := LibvirtAssert{} + t.Parallel() + DoTestKbsKeyRelease(t, testEnv, assert) +} diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index c5f65cfc12..beeecd5d71 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index e46562a7eb..42d23be370 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -81,7 +81,7 @@ type InstallOverlay interface { const PodWaitTimeout = time.Second * 30 // trustee repo related base path -const TRUSTEE_REPO_PATH = "../trustee" +const TRUSTEE_REPO_PATH = "trustee" func saveToFile(filename string, content []byte) error { // Save contents to file @@ -134,16 +134,23 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } defer keyOutputFile.Close() - _, privateKey, err := ed25519.GenerateKey(rand.Reader) + pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) log.Errorf("%v", err) return nil, err } + b, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ Type: "PRIVATE KEY", - Bytes: privateKey, + Bytes: b, }) // Save private key to file @@ -154,17 +161,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { return nil, err } - publicKey := privateKey.Public().(ed25519.PublicKey) - publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) + b, err = x509.MarshalPKIXPublicKey(pubKey) if err != nil { - err = fmt.Errorf("generating Ed25519 public key: %w\n", err) + err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err) log.Errorf("%v", err) return nil, err } publicKeyPEM := pem.EncodeToMemory(&pem.Block{ Type: "PUBLIC KEY", - Bytes: publicKeyX509, + Bytes: b, }) // Save public key to file @@ -335,6 +341,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf resources := client.Resources(namespace) + kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}} + fmt.Printf("Wait for the %s deployment be available\n", deploymentName) + if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*2)); err != nil { + return "", err + } + services := &corev1.ServiceList{} if err := resources.List(context.TODO(), services); err != nil { return "", err @@ -530,17 +543,7 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props } for ds, timeout := range daemonSetList { - // Wait for the daemonset to have at least one pod running then wait for each pod - // be ready. - - fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName()) - if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool { - ds = object.(*appsv1.DaemonSet) - - return ds.Status.CurrentNumberScheduled > 0 - }), wait.WithTimeout(time.Minute*5)); err != nil { - return err - } + // Wait for the daemonset to have at least one pod ready pods, err := GetDaemonSetOwnedPods(ctx, cfg, ds) if err != nil { return err