libvirt: enable attestation test cases

Fixes: #1825

Signed-off-by: Qi Feng Huo <huoqif@cn.ibm.com>

src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml

@@ -24,6 +24,7 @@ configMapGenerator:
   - LIBVIRT_POOL="default" # set
   - DISABLECVM="true" # set as false to enable confidential VM
   - SECURE_COMMS="false" # set as true to enable Secure Comms
+  - AA_KBC_PARAMS="" #set KBC params for podvm
   #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv
   #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2

src/cloud-api-adaptor/libvirt/README.md

@@ -215,6 +215,8 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2
 * ``TEST_PODVM_IMAGE`` - image to be used for this testing
 * ``CLOUD_PROVIDER`` - which cloud provider should be used
 * ``TEST_E2E_TIMEOUT`` - test timeout
+* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow
+* ``TEE_CUSTOMIZED_OPA`` - specify a customized OPA policy file for KBS service.
 * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh))
 
 # Delete Confidential Containers and cloud-api-adaptor from the cluster

src/cloud-api-adaptor/test/e2e/README.md

@@ -48,6 +48,27 @@ To leave the cluster untouched by the execution finish you should export `TEST_T
 
 To use existing cluster which have already installed Cloud API Adaptor, you should export `TEST_INSTALL_CAA=no`.
 
+## Attestation and KBS specific
+
+To deploy the KBS service and test attestation related cases, set `DEPLOY_KBS=yes`.
+To use a customized OPA policy file in KBS service, specify the OPA file via `TEE_CUSTOMIZED_OPA=/path/policy.rego`.
+For example, we need set `TEE_CUSTOMIZED_OPA=/$trustee_path/kbs/sample_policies/allow_all.rego` when test sample TEE as default policy disabled sample TEE.
+
+We also need artificats from trustee when do attestation test, to prepare trustee, do as following. 
+
+```
+pushd ${cloud-api-adaptor}/test/e2e
+git clone https://github.com/confidential-containers/trustee.git
+pushd trustee
+git checkout $COMMIT
+pushd kbs
+make CLI_FEATURES=sample_only cli
+popd
+popd
+popd
+```
+Note: we're using $COMMIT 65ee7e1acccd13dcb515058e71c5f8bfb4281e35 when writing this guide, it might change.
+
 ## Provision file specifics
 
 As mentioned on the previous section, a properties file can be passed to the cloud provisioner that will be used to controll the provisioning operations. The properties are specific of each cloud provider though, see on the sections below.

src/cloud-api-adaptor/test/e2e/azure_test.go

@@ -43,7 +43,7 @@ func TestCreateNginxDeploymentAzure(t *testing.T) {
 }
 
 func TestKbsKeyRelease(t *testing.T) {
-	if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" {
+	if !isTestWithKbs() {
 		t.Skip("Skipping kbs related test as kbs is not deployed")
 	}
 	t.Parallel()

src/cloud-api-adaptor/test/e2e/common.go

@@ -5,6 +5,7 @@ package e2e
 
 import (
 	"net"
+	"os"
 	"testing"
 	"time"
 
@@ -26,6 +27,14 @@ const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest"
 const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180
 const DEFAULT_AUTH_SECRET = "auth-json-secret-default"
 
+func isTestWithKbs() bool {
+	return os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes"
+}
+
+func isTestWithCustomizedOpa() bool {
+	return os.Getenv("TEE_CUSTOMIZED_OPA") != ""
+}
+
 type PodOption func(*corev1.Pod)
 
 func WithRestartPolicy(restartPolicy corev1.RestartPolicy) PodOption {

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -595,3 +595,26 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) {
 
 	NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run()
 }
+
+func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) {
+
+	log.Info("Do test kbs key release failure case")
+	pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget-failure")
+	testCommands := []TestCommand{
+		{
+			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"},
+			ContainerName: pod.Spec.Containers[0].Name,
+			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
+				if strings.Contains(stdout.String(), "request unautorized") {
+					log.Infof("Pass failure case as: %s", stdout.String())
+					return true
+				} else {
+					log.Errorf("Failed to faliure case as: %s", stdout.String())
+					return false
+				}
+			},
+		},
+	}
+
+	NewTestCase(t, e, "DoTestKbsKeyReleaseForFailure", assert, "Kbs key release is failed").WithPod(pod).WithTestCommands(testCommands).Run()
+}

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -96,3 +96,27 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) {
 	assert := LibvirtAssert{}
 	DoTestPodsMTLSCommunication(t, testEnv, assert)
 }
+
+func TestLibvirtKbsKeyReleaseWithDefaultOpa(t *testing.T) {
+	if !isTestWithKbs() {
+		t.Skip("Skipping kbs related test as kbs is not deployed")
+	}
+	if !isTestWithCustomizedOpa() {
+		t.Skip("Skipping TestLibvirtKbsKeyReleaseWithCustomizedOpa as default opa is used")
+	}
+	assert := LibvirtAssert{}
+	t.Parallel()
+	DoTestKbsKeyRelease(t, testEnv, assert)
+}
+
+func TestLibvirtKbsKeyReleaseWithCustomizedOpa(t *testing.T) {
+	if !isTestWithKbs() {
+		t.Skip("Skipping kbs related test as kbs is not deployed")
+	}
+	if isTestWithCustomizedOpa() {
+		t.Skip("Skipping TestLibvirtKbsKeyReleaseWithDefaultOpa as customized opa is used")
+	}
+	assert := LibvirtAssert{}
+	t.Parallel()
+	DoTestKbsKeyReleaseForFailure(t, testEnv, assert)
+}

src/cloud-api-adaptor/test/e2e/main_test.go

@@ -97,10 +97,14 @@ func TestMain(m *testing.M) {
 
 	// The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA
 	shouldDeployKbs := false
-	if os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" {
+	if isTestWithKbs() {
 		shouldDeployKbs = true
 	}
 
+	// The TEE_CUSTOMIZED_OPA is an optional variable which specifies the opa file path
+	// such as: $HOME/trustee/kbs/sample_policies/allow_all.rego.
+	customizedOpaFile := os.Getenv("TEE_CUSTOMIZED_OPA")
+
 	if !shouldProvisionCluster {
 		// Look for a suitable kubeconfig file in the sequence: --kubeconfig flag,
 		// or KUBECONFIG variable, or $HOME/.kube/config.
@@ -154,6 +158,15 @@ func TestMain(m *testing.M) {
 
 			kbsparams = "cc_kbc::http://" + kbsEndpoint
 			log.Infof("KBS PARAMS: %s", kbsparams)
+			if customizedOpaFile != "" {
+				log.Info("Enable customized opa file in KBS service.")
+				if _, err := os.Stat(customizedOpaFile); err != nil {
+					return ctx, err
+				}
+				if err = keyBrokerService.EnableKbsCustomizedPolicy("http://"+kbsEndpoint, customizedOpaFile); err != nil {
+					return ctx, err
+				}
+			}
 		}
 
 		if podvmImage != "" {

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
+		"network":       {"default", "LIBVIRT_NET"},
+		"storage":       {"default", "LIBVIRT_POOL"},
+		"pause_image":   {"", "PAUSE_IMAGE"},
+		"podvm_volume":  {"", "LIBVIRT_VOL_NAME"},
+		"uri":           {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
+		"vxlan_port":    {"", "VXLAN_PORT"},
+		"AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"},
 	}
 
 	for k, v := range mapProps {

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -81,7 +81,7 @@ type InstallOverlay interface {
 const PodWaitTimeout = time.Second * 30
 
 // trustee repo related base path
-const TRUSTEE_REPO_PATH = "../trustee"
+const TRUSTEE_REPO_PATH = "trustee"
 
 func saveToFile(filename string, content []byte) error {
 	// Save contents to file
@@ -134,16 +134,23 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		}
 		defer keyOutputFile.Close()
 
-		_, privateKey, err := ed25519.GenerateKey(rand.Reader)
+		pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
 			err = fmt.Errorf("generating Ed25519 key pair: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
+		b, err := x509.MarshalPKCS8PrivateKey(privateKey)
+		if err != nil {
+			err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err)
+			log.Errorf("%v", err)
+			return nil, err
+		}
+
 		privateKeyPEM := pem.EncodeToMemory(&pem.Block{
 			Type:  "PRIVATE KEY",
-			Bytes: privateKey,
+			Bytes: b,
 		})
 
 		// Save private key to file
@@ -154,17 +161,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 			return nil, err
 		}
 
-		publicKey := privateKey.Public().(ed25519.PublicKey)
-		publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey)
+		b, err = x509.MarshalPKIXPublicKey(pubKey)
 		if err != nil {
-			err = fmt.Errorf("generating Ed25519 public key: %w\n", err)
+			err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
 		publicKeyPEM := pem.EncodeToMemory(&pem.Block{
 			Type:  "PUBLIC KEY",
-			Bytes: publicKeyX509,
+			Bytes: b,
 		})
 
 		// Save public key to file
@@ -335,6 +341,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 
 	resources := client.Resources(namespace)
 
+	kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}}
+	fmt.Printf("Wait for the %s deployment be available\n", deploymentName)
+	if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue),
+		wait.WithTimeout(time.Minute*2)); err != nil {
+		return "", err
+	}
+
 	services := &corev1.ServiceList{}
 	if err := resources.List(context.TODO(), services); err != nil {
 		return "", err
@@ -368,6 +381,21 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 	return "", fmt.Errorf("Service %s not found", serviceName)
 }
 
+func (p *KeyBrokerService) EnableKbsCustomizedPolicy(kbsEndpoint string, customizedOpaFile string) error {
+	log.Info("EnableKbsCustomizedPolicy")
+	kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release")
+	privateKey := "../../kbs/config/kubernetes/base/kbs.key"
+	cmd := exec.Command("./kbs-client", "--url", kbsEndpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", customizedOpaFile)
+	cmd.Dir = kbsClientDir
+	cmd.Env = os.Environ()
+	stdoutStderr, err := cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error {
 	log.Info("Customize the overlay yaml file")
 	if err := p.installOverlay.Edit(ctx, cfg, props); err != nil {
@@ -533,14 +561,17 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 		// Wait for the daemonset to have at least one pod running then wait for each pod
 		// be ready.
 
-		fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName())
-		if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool {
-			ds = object.(*appsv1.DaemonSet)
-
-			return ds.Status.CurrentNumberScheduled > 0
-		}), wait.WithTimeout(time.Minute*5)); err != nil {
-			return err
-		}
+		//fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName())
+		//if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool {
+		//	ds = object.(*appsv1.DaemonSet)
+		//
+		//	return ds.Status.CurrentNumberScheduled > 0
+		//}), wait.WithTimeout(time.Minute*5)); err != nil {
+		//	return err
+		//}
+
+		// wait for each pod be ready for each ds, which is good enough. 
+		// Leave the useless ds status waiting as commented, we cam remove it later.
 		pods, err := GetDaemonSetOwnedPods(ctx, cfg, ds)
 		if err != nil {
 			return err