From ee5f9d4709e6eb672bfacf22a50566e1f9b26c20 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Wed, 8 May 2024 09:34:57 +0800 Subject: [PATCH] libvirt: enable attestation test cases Fixes: #1825 Signed-off-by: Qi Feng Huo --- .../overlays/libvirt/kustomization.yaml | 1 + src/cloud-api-adaptor/libvirt/README.md | 2 + src/cloud-api-adaptor/test/e2e/README.md | 21 +++++++ src/cloud-api-adaptor/test/e2e/azure_test.go | 2 +- src/cloud-api-adaptor/test/e2e/common.go | 9 +++ .../test/e2e/common_suite.go | 23 +++++++ .../test/e2e/libvirt_test.go | 24 ++++++++ src/cloud-api-adaptor/test/e2e/main_test.go | 15 ++++- .../provisioner/libvirt/provision_common.go | 13 ++-- .../test/provisioner/provision.go | 61 ++++++++++++++----- 10 files changed, 148 insertions(+), 23 deletions(-) diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 3b6d6aa244..a96535d5ba 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -24,6 +24,7 @@ configMapGenerator: - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM - SECURE_COMMS="false" # set as true to enable Secure Comms + - AA_KBC_PARAMS="" #set KBC params for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/libvirt/README.md b/src/cloud-api-adaptor/libvirt/README.md index 34cd7591ac..3fb34cfaa0 100644 --- a/src/cloud-api-adaptor/libvirt/README.md +++ b/src/cloud-api-adaptor/libvirt/README.md @@ -215,6 +215,8 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2 * ``TEST_PODVM_IMAGE`` - image to be used for this testing * ``CLOUD_PROVIDER`` - which cloud provider should be used * ``TEST_E2E_TIMEOUT`` - test timeout +* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow +* ``TEE_CUSTOMIZED_OPA`` - specify a customized OPA policy file for KBS service. * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh)) # Delete Confidential Containers and cloud-api-adaptor from the cluster diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 03432c2a1f..20cf475f53 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -48,6 +48,27 @@ To leave the cluster untouched by the execution finish you should export `TEST_T To use existing cluster which have already installed Cloud API Adaptor, you should export `TEST_INSTALL_CAA=no`. +## Attestation and KBS specific + +To deploy the KBS service and test attestation related cases, set `DEPLOY_KBS=yes`. +To use a customized OPA policy file in KBS service, specify the OPA file via `TEE_CUSTOMIZED_OPA=/path/policy.rego`. +For example, we need set `TEE_CUSTOMIZED_OPA=/$trustee_path/kbs/sample_policies/allow_all.rego` when test sample TEE as default policy disabled sample TEE. + +We also need artificats from trustee when do attestation test, to prepare trustee, do as following. + +``` +pushd ${cloud-api-adaptor}/test/e2e +git clone https://github.com/confidential-containers/trustee.git +pushd trustee +git checkout $COMMIT +pushd kbs +make CLI_FEATURES=sample_only cli +popd +popd +popd +``` +Note: we're using $COMMIT 65ee7e1acccd13dcb515058e71c5f8bfb4281e35 when writing this guide, it might change. + ## Provision file specifics As mentioned on the previous section, a properties file can be passed to the cloud provisioner that will be used to controll the provisioning operations. The properties are specific of each cloud provider though, see on the sections below. diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index 5330a5b217..ee60f06b66 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -43,7 +43,7 @@ func TestCreateNginxDeploymentAzure(t *testing.T) { } func TestKbsKeyRelease(t *testing.T) { - if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" { + if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } t.Parallel() diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index 1504a82ded..66e369ec7a 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -5,6 +5,7 @@ package e2e import ( "net" + "os" "testing" "time" @@ -26,6 +27,14 @@ const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest" const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180 const DEFAULT_AUTH_SECRET = "auth-json-secret-default" +func isTestWithKbs() bool { + return os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" +} + +func isTestWithCustomizedOpa() bool { + return os.Getenv("TEE_CUSTOMIZED_OPA") != "" +} + type PodOption func(*corev1.Pod) func WithRestartPolicy(restartPolicy corev1.RestartPolicy) PodOption { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 33b567d004..4f491890c9 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -595,3 +595,26 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run() } + +func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) { + + log.Info("Do test kbs key release failure case") + pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget-failure") + testCommands := []TestCommand{ + { + Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, + ContainerName: pod.Spec.Containers[0].Name, + TestCommandStdoutFn: func(stdout bytes.Buffer) bool { + if strings.Contains(stdout.String(), "request unautorized") { + log.Infof("Pass failure case as: %s", stdout.String()) + return true + } else { + log.Errorf("Failed to faliure case as: %s", stdout.String()) + return false + } + }, + }, + } + + NewTestCase(t, e, "DoTestKbsKeyReleaseForFailure", assert, "Kbs key release is failed").WithPod(pod).WithTestCommands(testCommands).Run() +} diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 3be18c9daf..6e2ae2c37d 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -96,3 +96,27 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) { assert := LibvirtAssert{} DoTestPodsMTLSCommunication(t, testEnv, assert) } + +func TestLibvirtKbsKeyReleaseWithDefaultOpa(t *testing.T) { + if !isTestWithKbs() { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + if !isTestWithCustomizedOpa() { + t.Skip("Skipping TestLibvirtKbsKeyReleaseWithCustomizedOpa as default opa is used") + } + assert := LibvirtAssert{} + t.Parallel() + DoTestKbsKeyRelease(t, testEnv, assert) +} + +func TestLibvirtKbsKeyReleaseWithCustomizedOpa(t *testing.T) { + if !isTestWithKbs() { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + if isTestWithCustomizedOpa() { + t.Skip("Skipping TestLibvirtKbsKeyReleaseWithDefaultOpa as customized opa is used") + } + assert := LibvirtAssert{} + t.Parallel() + DoTestKbsKeyReleaseForFailure(t, testEnv, assert) +} \ No newline at end of file diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 246fe996bc..acb53c5c94 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -97,10 +97,14 @@ func TestMain(m *testing.M) { // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA shouldDeployKbs := false - if os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" { + if isTestWithKbs() { shouldDeployKbs = true } + // The TEE_CUSTOMIZED_OPA is an optional variable which specifies the opa file path + // such as: $HOME/trustee/kbs/sample_policies/allow_all.rego. + customizedOpaFile := os.Getenv("TEE_CUSTOMIZED_OPA") + if !shouldProvisionCluster { // Look for a suitable kubeconfig file in the sequence: --kubeconfig flag, // or KUBECONFIG variable, or $HOME/.kube/config. @@ -154,6 +158,15 @@ func TestMain(m *testing.M) { kbsparams = "cc_kbc::http://" + kbsEndpoint log.Infof("KBS PARAMS: %s", kbsparams) + if customizedOpaFile != "" { + log.Info("Enable customized opa file in KBS service.") + if _, err := os.Stat(customizedOpaFile); err != nil { + return ctx, err + } + if err = keyBrokerService.EnableKbsCustomizedPolicy("http://"+kbsEndpoint, customizedOpaFile); err != nil { + return ctx, err + } + } } if podvmImage != "" { diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index c5f65cfc12..beeecd5d71 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index e46562a7eb..d69ca2f435 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -81,7 +81,7 @@ type InstallOverlay interface { const PodWaitTimeout = time.Second * 30 // trustee repo related base path -const TRUSTEE_REPO_PATH = "../trustee" +const TRUSTEE_REPO_PATH = "trustee" func saveToFile(filename string, content []byte) error { // Save contents to file @@ -134,16 +134,23 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } defer keyOutputFile.Close() - _, privateKey, err := ed25519.GenerateKey(rand.Reader) + pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) log.Errorf("%v", err) return nil, err } + b, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ Type: "PRIVATE KEY", - Bytes: privateKey, + Bytes: b, }) // Save private key to file @@ -154,17 +161,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { return nil, err } - publicKey := privateKey.Public().(ed25519.PublicKey) - publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) + b, err = x509.MarshalPKIXPublicKey(pubKey) if err != nil { - err = fmt.Errorf("generating Ed25519 public key: %w\n", err) + err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err) log.Errorf("%v", err) return nil, err } publicKeyPEM := pem.EncodeToMemory(&pem.Block{ Type: "PUBLIC KEY", - Bytes: publicKeyX509, + Bytes: b, }) // Save public key to file @@ -335,6 +341,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf resources := client.Resources(namespace) + kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}} + fmt.Printf("Wait for the %s deployment be available\n", deploymentName) + if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*2)); err != nil { + return "", err + } + services := &corev1.ServiceList{} if err := resources.List(context.TODO(), services); err != nil { return "", err @@ -368,6 +381,21 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf return "", fmt.Errorf("Service %s not found", serviceName) } +func (p *KeyBrokerService) EnableKbsCustomizedPolicy(kbsEndpoint string, customizedOpaFile string) error { + log.Info("EnableKbsCustomizedPolicy") + kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") + privateKey := "../../kbs/config/kubernetes/base/kbs.key" + cmd := exec.Command("./kbs-client", "--url", kbsEndpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", customizedOpaFile) + cmd.Dir = kbsClientDir + cmd.Env = os.Environ() + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + return nil +} + func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { log.Info("Customize the overlay yaml file") if err := p.installOverlay.Edit(ctx, cfg, props); err != nil { @@ -533,14 +561,17 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props // Wait for the daemonset to have at least one pod running then wait for each pod // be ready. - fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName()) - if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool { - ds = object.(*appsv1.DaemonSet) - - return ds.Status.CurrentNumberScheduled > 0 - }), wait.WithTimeout(time.Minute*5)); err != nil { - return err - } + //fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName()) + //if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool { + // ds = object.(*appsv1.DaemonSet) + // + // return ds.Status.CurrentNumberScheduled > 0 + //}), wait.WithTimeout(time.Minute*5)); err != nil { + // return err + //} + + // wait for each pod be ready for each ds, which is good enough. + // Leave the useless ds status waiting as commented, we cam remove it later. pods, err := GetDaemonSetOwnedPods(ctx, cfg, ds) if err != nil { return err