libvirt: e2e test for attestation for sample tee

src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml

@@ -24,6 +24,7 @@ configMapGenerator:
   - LIBVIRT_POOL="default" # set
   - DISABLECVM="true" # set as false to enable confidential VM
   - SECURE_COMMS="false" # set as true to enable Secure Comms
+  - AA_KBC_PARAMS="" #set KBC params for podvm
   #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv
   #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2

src/cloud-api-adaptor/libvirt/README.md

@@ -215,6 +215,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2
 * ``TEST_PODVM_IMAGE`` - image to be used for this testing
 * ``CLOUD_PROVIDER`` - which cloud provider should be used
 * ``TEST_E2E_TIMEOUT`` - test timeout
+* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow
 * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh))
 
 # Delete Confidential Containers and cloud-api-adaptor from the cluster

src/cloud-api-adaptor/podvm/Makefile.inc

@@ -23,7 +23,7 @@ ARCH        := $(or $(ARCH),$(HOST_ARCH))
 # Normalise x86_64 / amd64 for input ARCH
 ARCH        := $(subst amd64,x86_64,$(ARCH))
 DEB_ARCH    := $(subst x86_64,amd64,$(ARCH))
-AA_KBC      ?= offline_fs_kbc
+AA_KBC      ?= cc_kbc
 KBC_URI     ?= null
 LIBC        ?= $(if $(filter $(ARCH),s390x ppc64le),gnu,musl)
 RUST_ARCH   ?= $(subst ppc64le,powerpc64le,$(ARCH))

src/cloud-api-adaptor/test/e2e/README.md

@@ -48,6 +48,37 @@ To leave the cluster untouched by the execution finish you should export `TEST_T
 
 To use existing cluster which have already installed Cloud API Adaptor, you should export `TEST_INSTALL_CAA=no`.
 
+## Attestation and KBS specific
+
+We need artificats from trustee when do attestation test, to prepare trustee, do as following. 
+
+```
+pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test
+git clone https://github.com/confidential-containers/trustee.git
+pushd trustee
+git checkout $(../../hack/yq-shim.sh '.git.kbs.reference' ../../versions.yaml)
+pushd kbs
+make CLI_FEATURES=sample_only cli
+popd
+popd
+popd
+```
+
+We need build and use the PodVM image with `AA_KBC=cc_kbc` enabled, for example:
+```
+pushd ${cloud-api-adaptor}
+AA_KBC=cc_kbc make podvm-builder podvm-binaries podvm-image
+popd
+```
+Then extract the PodVM image and use it following [extracting-the-qcow2-image](../../podvm/README.md#extracting-the-qcow2-image)
+
+To deploy the KBS service and test attestation related cases, export following variables like:
+```
+export DEPLOY_KBS=yes
+export KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml)
+export KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml)
+````
+
 ## Provision file specifics
 
 As mentioned on the previous section, a properties file can be passed to the cloud provisioner that will be used to controll the provisioning operations. The properties are specific of each cloud provider though, see on the sections below.

src/cloud-api-adaptor/test/e2e/azure_test.go

@@ -6,7 +6,6 @@
 package e2e
 
 import (
-	"os"
 	"testing"
 
 	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/azure"
@@ -43,9 +42,13 @@ func TestCreateNginxDeploymentAzure(t *testing.T) {
 }
 
 func TestKbsKeyRelease(t *testing.T) {
-	if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" {
+	if !isTestWithKbs() {
 		t.Skip("Skipping kbs related test as kbs is not deployed")
 	}
 	t.Parallel()
 	DoTestKbsKeyRelease(t, testEnv, assert)
+
+	// @Magnus @Kartik, are you going to enable this negative test for azure?
+	// _ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego")
+	// DoTestKbsKeyReleaseForFailure(t, testEnv, assert)
 }

src/cloud-api-adaptor/test/e2e/common.go

@@ -5,6 +5,7 @@ package e2e
 
 import (
 	"net"
+	"os"
 	"testing"
 	"time"
 
@@ -26,6 +27,10 @@ const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest"
 const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180
 const DEFAULT_AUTH_SECRET = "auth-json-secret-default"
 
+func isTestWithKbs() bool {
+	return os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes"
+}
+
 type PodOption func(*corev1.Pod)
 
 func WithRestartPolicy(restartPolicy corev1.RestartPolicy) PodOption {

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -573,6 +573,8 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs
 
 }
 
+// DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaning opa in kbs
+// as test cases might be run in parallel
 func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) {
 
 	log.Info("Do test kbs key release")
@@ -595,3 +597,28 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) {
 
 	NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run()
 }
+
+// DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaning opa in kbs
+// as test cases might be run in parallel
+func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) {
+
+	log.Info("Do test kbs key release failure case")
+	pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget-failure")
+	testCommands := []TestCommand{
+		{
+			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"},
+			ContainerName: pod.Spec.Containers[0].Name,
+			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
+				if strings.Contains(stdout.String(), "request unautorized") {
+					log.Infof("Pass failure case as: %s", stdout.String())
+					return true
+				} else {
+					log.Errorf("Failed to faliure case as: %s", stdout.String())
+					return false
+				}
+			},
+		},
+	}
+
+	NewTestCase(t, e, "DoTestKbsKeyReleaseForFailure", assert, "Kbs key release is failed").WithPod(pod).WithTestCommands(testCommands).Run()
+}

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -96,3 +96,15 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) {
 	assert := LibvirtAssert{}
 	DoTestPodsMTLSCommunication(t, testEnv, assert)
 }
+
+func TestLibvirtKbsKeyRelease(t *testing.T) {
+	if !isTestWithKbs() {
+		t.Skip("Skipping kbs related test as kbs is not deployed")
+	}
+	_ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego")
+	assert := LibvirtAssert{}
+	t.Parallel()
+	DoTestKbsKeyReleaseForFailure(t, testEnv, assert)
+	_ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego")
+	DoTestKbsKeyRelease(t, testEnv, assert)
+}

src/cloud-api-adaptor/test/e2e/main_test.go

@@ -96,10 +96,7 @@ func TestMain(m *testing.M) {
 	}
 
 	// The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA
-	shouldDeployKbs := false
-	if os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" {
-		shouldDeployKbs = true
-	}
+	shouldDeployKbs := isTestWithKbs()
 
 	if !shouldProvisionCluster {
 		// Look for a suitable kubeconfig file in the sequence: --kubeconfig flag,
@@ -152,7 +149,7 @@ func TestMain(m *testing.M) {
 				return ctx, err
 			}
 
-			kbsparams = "cc_kbc::http://" + kbsEndpoint
+			kbsparams = "cc_kbc::" + kbsEndpoint
 			log.Infof("KBS PARAMS: %s", kbsparams)
 		}
 

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
+		"network":       {"default", "LIBVIRT_NET"},
+		"storage":       {"default", "LIBVIRT_POOL"},
+		"pause_image":   {"", "PAUSE_IMAGE"},
+		"podvm_volume":  {"", "LIBVIRT_VOL_NAME"},
+		"uri":           {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"},
+		"vxlan_port":    {"", "VXLAN_PORT"},
+		"AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"},
 	}
 
 	for k, v := range mapProps {

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -63,6 +63,7 @@ type NewInstallOverlayFunc func(installDir, provider string) (InstallOverlay, er
 
 type KeyBrokerService struct {
 	installOverlay InstallOverlay // Pointer to the kustomize overlay
+	endpoint       string         // KBS Service endpoint, such as: http://NodeIP:Port
 }
 
 var NewInstallOverlayFunctions = make(map[string]NewInstallOverlayFunc)
@@ -134,16 +135,23 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		}
 		defer keyOutputFile.Close()
 
-		_, privateKey, err := ed25519.GenerateKey(rand.Reader)
+		pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
 			err = fmt.Errorf("generating Ed25519 key pair: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
+		b, err := x509.MarshalPKCS8PrivateKey(privateKey)
+		if err != nil {
+			err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err)
+			log.Errorf("%v", err)
+			return nil, err
+		}
+
 		privateKeyPEM := pem.EncodeToMemory(&pem.Block{
 			Type:  "PRIVATE KEY",
-			Bytes: privateKey,
+			Bytes: b,
 		})
 
 		// Save private key to file
@@ -154,17 +162,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 			return nil, err
 		}
 
-		publicKey := privateKey.Public().(ed25519.PublicKey)
-		publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey)
+		b, err = x509.MarshalPKIXPublicKey(pubKey)
 		if err != nil {
-			err = fmt.Errorf("generating Ed25519 public key: %w\n", err)
+			err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
 		publicKeyPEM := pem.EncodeToMemory(&pem.Block{
 			Type:  "PUBLIC KEY",
-			Bytes: publicKeyX509,
+			Bytes: b,
 		})
 
 		// Save public key to file
@@ -184,6 +191,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 
 	return &KeyBrokerService{
 		installOverlay: overlay,
+		endpoint:       "",
 	}, nil
 }
 
@@ -335,6 +343,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 
 	resources := client.Resources(namespace)
 
+	kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}}
+	fmt.Printf("Wait for the %s deployment be available\n", deploymentName)
+	if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue),
+		wait.WithTimeout(time.Minute*2)); err != nil {
+		return "", err
+	}
+
 	services := &corev1.ServiceList{}
 	if err := resources.List(context.TODO(), services); err != nil {
 		return "", err
@@ -360,14 +375,30 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf
 				return "", err
 			}
 
-			serviceEndpoint := fmt.Sprintf("%s:%d", nodeIP, nodePort)
-			return serviceEndpoint, nil
+			p.endpoint = fmt.Sprintf("http://%s:%d", nodeIP, nodePort)
+			return p.endpoint, nil
 		}
 	}
 
 	return "", fmt.Errorf("Service %s not found", serviceName)
 }
 
+func (p *KeyBrokerService) EnableKbsCustomizedPolicy(customizedOpaFile string) error {
+	kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release")
+	privateKey := "../../kbs/config/kubernetes/base/kbs.key"
+	policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile)
+	log.Info("EnableKbsCustomizedPolicy: ", policyFile)
+	cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", policyFile)
+	cmd.Dir = kbsClientDir
+	cmd.Env = os.Environ()
+	stdoutStderr, err := cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error {
 	log.Info("Customize the overlay yaml file")
 	if err := p.installOverlay.Edit(ctx, cfg, props); err != nil {
@@ -536,7 +567,6 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 		fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName())
 		if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool {
 			ds = object.(*appsv1.DaemonSet)
-
 			return ds.Status.CurrentNumberScheduled > 0
 		}), wait.WithTimeout(time.Minute*5)); err != nil {
 			return err

src/cloud-api-adaptor/test/tools/provisioner-cli/main.go

@@ -109,7 +109,7 @@ func main() {
 				log.Fatal(err)
 			}
 
-			kbsparams := "cc_kbc::http://" + kbsEndpoint
+			kbsparams := "cc_kbc::" + kbsEndpoint
 			log.Infof("KBS PARAMS: %s", kbsparams)
 
 			props = provisioner.GetProperties(context.TODO(), cfg)