From 9a7b09b528ba09c1729e6e6dc75ef51ba960f1c5 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Wed, 8 May 2024 09:34:57 +0800 Subject: [PATCH 1/7] libvirt: enable attestation test cases Fixes: #1825 Signed-off-by: Qi Feng Huo --- .../overlays/libvirt/kustomization.yaml | 1 + src/cloud-api-adaptor/libvirt/README.md | 2 + src/cloud-api-adaptor/test/e2e/README.md | 20 +++++++++ src/cloud-api-adaptor/test/e2e/azure_test.go | 3 +- src/cloud-api-adaptor/test/e2e/common.go | 9 ++++ .../test/e2e/common_suite.go | 23 ++++++++++ .../test/e2e/libvirt_test.go | 24 ++++++++++ src/cloud-api-adaptor/test/e2e/main_test.go | 15 ++++++- .../provisioner/libvirt/provision_common.go | 13 +++--- .../test/provisioner/provision.go | 44 +++++++++++++++---- 10 files changed, 137 insertions(+), 17 deletions(-) diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 3b6d6aa24..a96535d5b 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -24,6 +24,7 @@ configMapGenerator: - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM - SECURE_COMMS="false" # set as true to enable Secure Comms + - AA_KBC_PARAMS="" #set KBC params for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/libvirt/README.md b/src/cloud-api-adaptor/libvirt/README.md index 34cd7591a..3fb34cfaa 100644 --- a/src/cloud-api-adaptor/libvirt/README.md +++ b/src/cloud-api-adaptor/libvirt/README.md @@ -215,6 +215,8 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2 * ``TEST_PODVM_IMAGE`` - image to be used for this testing * ``CLOUD_PROVIDER`` - which cloud provider should be used * ``TEST_E2E_TIMEOUT`` - test timeout +* ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow +* ``TEE_CUSTOMIZED_OPA`` - specify a customized OPA policy file for KBS service. * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh)) # Delete Confidential Containers and cloud-api-adaptor from the cluster diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 03432c2a1..b4da4182f 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -48,6 +48,26 @@ To leave the cluster untouched by the execution finish you should export `TEST_T To use existing cluster which have already installed Cloud API Adaptor, you should export `TEST_INSTALL_CAA=no`. +## Attestation and KBS specific + +We need artificats from trustee when do attestation test, to prepare trustee, do as following. + +``` +pushd ${cloud-api-adaptor}/test/e2e +git clone https://github.com/confidential-containers/trustee.git +pushd trustee +git checkout $(../../../hack/yq-shim.sh '.git.kbs.reference' ../../../versions.yaml) +pushd kbs +make CLI_FEATURES=sample_only cli +popd +popd +popd +``` + +To deploy the KBS service and test attestation related cases, set `DEPLOY_KBS=yes`. +To use a customized OPA policy file in KBS service, specify the OPA file via `TEE_CUSTOMIZED_OPA=/path/policy.rego`. +For example, we need set `TEE_CUSTOMIZED_OPA=/$trustee_path/kbs/sample_policies/allow_all.rego` when test sample TEE as default policy disabled sample TEE. + ## Provision file specifics As mentioned on the previous section, a properties file can be passed to the cloud provisioner that will be used to controll the provisioning operations. The properties are specific of each cloud provider though, see on the sections below. diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index 5330a5b21..07953f3b1 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -6,7 +6,6 @@ package e2e import ( - "os" "testing" _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/azure" @@ -43,7 +42,7 @@ func TestCreateNginxDeploymentAzure(t *testing.T) { } func TestKbsKeyRelease(t *testing.T) { - if os.Getenv("DEPLOY_KBS") == "false" || os.Getenv("DEPLOY_KBS") == "no" { + if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } t.Parallel() diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index 1504a82de..66e369ec7 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -5,6 +5,7 @@ package e2e import ( "net" + "os" "testing" "time" @@ -26,6 +27,14 @@ const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest" const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180 const DEFAULT_AUTH_SECRET = "auth-json-secret-default" +func isTestWithKbs() bool { + return os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" +} + +func isTestWithCustomizedOpa() bool { + return os.Getenv("TEE_CUSTOMIZED_OPA") != "" +} + type PodOption func(*corev1.Pod) func WithRestartPolicy(restartPolicy corev1.RestartPolicy) PodOption { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 33b567d00..4f491890c 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -595,3 +595,26 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run() } + +func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) { + + log.Info("Do test kbs key release failure case") + pod := NewBusyboxPodWithName(E2eNamespace, "busybox-wget-failure") + testCommands := []TestCommand{ + { + Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, + ContainerName: pod.Spec.Containers[0].Name, + TestCommandStdoutFn: func(stdout bytes.Buffer) bool { + if strings.Contains(stdout.String(), "request unautorized") { + log.Infof("Pass failure case as: %s", stdout.String()) + return true + } else { + log.Errorf("Failed to faliure case as: %s", stdout.String()) + return false + } + }, + }, + } + + NewTestCase(t, e, "DoTestKbsKeyReleaseForFailure", assert, "Kbs key release is failed").WithPod(pod).WithTestCommands(testCommands).Run() +} diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 3be18c9da..d6198ac50 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -96,3 +96,27 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) { assert := LibvirtAssert{} DoTestPodsMTLSCommunication(t, testEnv, assert) } + +func TestLibvirtKbsKeyReleaseWithDefaultOpa(t *testing.T) { + if !isTestWithKbs() { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + if !isTestWithCustomizedOpa() { + t.Skip("Skipping TestLibvirtKbsKeyReleaseWithCustomizedOpa as default opa is used") + } + assert := LibvirtAssert{} + t.Parallel() + DoTestKbsKeyRelease(t, testEnv, assert) +} + +func TestLibvirtKbsKeyReleaseWithCustomizedOpa(t *testing.T) { + if !isTestWithKbs() { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + if isTestWithCustomizedOpa() { + t.Skip("Skipping TestLibvirtKbsKeyReleaseWithDefaultOpa as customized opa is used") + } + assert := LibvirtAssert{} + t.Parallel() + DoTestKbsKeyReleaseForFailure(t, testEnv, assert) +} diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 246fe996b..acb53c5c9 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -97,10 +97,14 @@ func TestMain(m *testing.M) { // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA shouldDeployKbs := false - if os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" { + if isTestWithKbs() { shouldDeployKbs = true } + // The TEE_CUSTOMIZED_OPA is an optional variable which specifies the opa file path + // such as: $HOME/trustee/kbs/sample_policies/allow_all.rego. + customizedOpaFile := os.Getenv("TEE_CUSTOMIZED_OPA") + if !shouldProvisionCluster { // Look for a suitable kubeconfig file in the sequence: --kubeconfig flag, // or KUBECONFIG variable, or $HOME/.kube/config. @@ -154,6 +158,15 @@ func TestMain(m *testing.M) { kbsparams = "cc_kbc::http://" + kbsEndpoint log.Infof("KBS PARAMS: %s", kbsparams) + if customizedOpaFile != "" { + log.Info("Enable customized opa file in KBS service.") + if _, err := os.Stat(customizedOpaFile); err != nil { + return ctx, err + } + if err = keyBrokerService.EnableKbsCustomizedPolicy("http://"+kbsEndpoint, customizedOpaFile); err != nil { + return ctx, err + } + } } if podvmImage != "" { diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index c5f65cfc1..beeecd5d7 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -312,12 +312,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index e46562a7e..756cba8f1 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -81,7 +81,7 @@ type InstallOverlay interface { const PodWaitTimeout = time.Second * 30 // trustee repo related base path -const TRUSTEE_REPO_PATH = "../trustee" +const TRUSTEE_REPO_PATH = "trustee" func saveToFile(filename string, content []byte) error { // Save contents to file @@ -134,16 +134,23 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } defer keyOutputFile.Close() - _, privateKey, err := ed25519.GenerateKey(rand.Reader) + pubKey, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) log.Errorf("%v", err) return nil, err } + b, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + err = fmt.Errorf("MarshalPKCS8PrivateKey private key: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ Type: "PRIVATE KEY", - Bytes: privateKey, + Bytes: b, }) // Save private key to file @@ -154,17 +161,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { return nil, err } - publicKey := privateKey.Public().(ed25519.PublicKey) - publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) + b, err = x509.MarshalPKIXPublicKey(pubKey) if err != nil { - err = fmt.Errorf("generating Ed25519 public key: %w\n", err) + err = fmt.Errorf("MarshalPKIXPublicKey Ed25519 public key: %w\n", err) log.Errorf("%v", err) return nil, err } publicKeyPEM := pem.EncodeToMemory(&pem.Block{ Type: "PUBLIC KEY", - Bytes: publicKeyX509, + Bytes: b, }) // Save public key to file @@ -335,6 +341,13 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf resources := client.Resources(namespace) + kbsDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespace}} + fmt.Printf("Wait for the %s deployment be available\n", deploymentName) + if err = wait.For(conditions.New(resources).DeploymentConditionMatch(kbsDeployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*2)); err != nil { + return "", err + } + services := &corev1.ServiceList{} if err := resources.List(context.TODO(), services); err != nil { return "", err @@ -368,6 +381,21 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf return "", fmt.Errorf("Service %s not found", serviceName) } +func (p *KeyBrokerService) EnableKbsCustomizedPolicy(kbsEndpoint string, customizedOpaFile string) error { + log.Info("EnableKbsCustomizedPolicy") + kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") + privateKey := "../../kbs/config/kubernetes/base/kbs.key" + cmd := exec.Command("./kbs-client", "--url", kbsEndpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", customizedOpaFile) + cmd.Dir = kbsClientDir + cmd.Env = os.Environ() + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + return nil +} + func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { log.Info("Customize the overlay yaml file") if err := p.installOverlay.Edit(ctx, cfg, props); err != nil { @@ -536,7 +564,7 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName()) if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool { ds = object.(*appsv1.DaemonSet) - + return ds.Status.CurrentNumberScheduled > 0 }), wait.WithTimeout(time.Minute*5)); err != nil { return err From 2fe33433130c2a3e1d9fff23a2cabf2741d28a84 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Wed, 8 May 2024 18:30:01 +0800 Subject: [PATCH 2/7] libvirt: enable attestation test cases and fix comments Fixes: #1825 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/test/e2e/README.md | 6 +++--- src/cloud-api-adaptor/test/e2e/common.go | 6 +++++- src/cloud-api-adaptor/test/e2e/main_test.go | 5 +---- src/cloud-api-adaptor/test/provisioner/provision.go | 1 - 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index b4da4182f..6e1527cc7 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -64,9 +64,9 @@ popd popd ``` -To deploy the KBS service and test attestation related cases, set `DEPLOY_KBS=yes`. -To use a customized OPA policy file in KBS service, specify the OPA file via `TEE_CUSTOMIZED_OPA=/path/policy.rego`. -For example, we need set `TEE_CUSTOMIZED_OPA=/$trustee_path/kbs/sample_policies/allow_all.rego` when test sample TEE as default policy disabled sample TEE. +To deploy the KBS service and test attestation related cases, set `export DEPLOY_KBS=yes`. +To use a customized OPA policy file in KBS service, specify the OPA file via `export TEE_CUSTOMIZED_OPA=/path/policy.rego`. +For example, we need set `export TEE_CUSTOMIZED_OPA=/$trustee_path/kbs/sample_policies/allow_all.rego` when test sample TEE as default policy disabled sample TEE. ## Provision file specifics diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index 66e369ec7..b648011ef 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -32,7 +32,11 @@ func isTestWithKbs() bool { } func isTestWithCustomizedOpa() bool { - return os.Getenv("TEE_CUSTOMIZED_OPA") != "" + if _, err := os.Stat(os.Getenv("TEE_CUSTOMIZED_OPA")); err != nil { + log.Printf("Stat TEE_CUSTOMIZED_OPA error: %v", err) + return false + } + return true } type PodOption func(*corev1.Pod) diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index acb53c5c9..7a3515aa5 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -96,10 +96,7 @@ func TestMain(m *testing.M) { } // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA - shouldDeployKbs := false - if isTestWithKbs() { - shouldDeployKbs = true - } + shouldDeployKbs := isTestWithKbs() // The TEE_CUSTOMIZED_OPA is an optional variable which specifies the opa file path // such as: $HOME/trustee/kbs/sample_policies/allow_all.rego. diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 756cba8f1..acf0602f6 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -564,7 +564,6 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props fmt.Printf("Wait for the %s DaemonSet be available\n", ds.GetName()) if err = wait.For(conditions.New(resources).ResourceMatch(ds, func(object k8s.Object) bool { ds = object.(*appsv1.DaemonSet) - return ds.Status.CurrentNumberScheduled > 0 }), wait.WithTimeout(time.Minute*5)); err != nil { return err From 1698635669dcf12d2f444e2b2bc598e7627f3b4a Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Thu, 9 May 2024 09:52:29 +0800 Subject: [PATCH 3/7] libvirt: enable attestation test cases and run negative test also Fixes: #1825 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/libvirt/README.md | 1 - src/cloud-api-adaptor/test/e2e/README.md | 2 -- src/cloud-api-adaptor/test/e2e/azure_test.go | 4 ++++ src/cloud-api-adaptor/test/e2e/common.go | 8 -------- .../test/e2e/common_suite.go | 4 ++++ .../test/e2e/libvirt_test.go | 20 ++++--------------- src/cloud-api-adaptor/test/e2e/main_test.go | 15 +------------- .../test/provisioner/provision.go | 13 +++++++----- .../test/tools/provisioner-cli/main.go | 2 +- 9 files changed, 22 insertions(+), 47 deletions(-) diff --git a/src/cloud-api-adaptor/libvirt/README.md b/src/cloud-api-adaptor/libvirt/README.md index 3fb34cfaa..aa33a078f 100644 --- a/src/cloud-api-adaptor/libvirt/README.md +++ b/src/cloud-api-adaptor/libvirt/README.md @@ -216,7 +216,6 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2 * ``CLOUD_PROVIDER`` - which cloud provider should be used * ``TEST_E2E_TIMEOUT`` - test timeout * ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow -* ``TEE_CUSTOMIZED_OPA`` - specify a customized OPA policy file for KBS service. * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh)) # Delete Confidential Containers and cloud-api-adaptor from the cluster diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 6e1527cc7..0bc9bfdcd 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -65,8 +65,6 @@ popd ``` To deploy the KBS service and test attestation related cases, set `export DEPLOY_KBS=yes`. -To use a customized OPA policy file in KBS service, specify the OPA file via `export TEE_CUSTOMIZED_OPA=/path/policy.rego`. -For example, we need set `export TEE_CUSTOMIZED_OPA=/$trustee_path/kbs/sample_policies/allow_all.rego` when test sample TEE as default policy disabled sample TEE. ## Provision file specifics diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index 07953f3b1..d22aed8cb 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -47,4 +47,8 @@ func TestKbsKeyRelease(t *testing.T) { } t.Parallel() DoTestKbsKeyRelease(t, testEnv, assert) + + // @Magnus @Kartik, are you going to enable this negative test for azure? + // _ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego") + // DoTestKbsKeyReleaseForFailure(t, testEnv, assert) } diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index b648011ef..62d663a87 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -31,14 +31,6 @@ func isTestWithKbs() bool { return os.Getenv("DEPLOY_KBS") == "true" || os.Getenv("DEPLOY_KBS") == "yes" } -func isTestWithCustomizedOpa() bool { - if _, err := os.Stat(os.Getenv("TEE_CUSTOMIZED_OPA")); err != nil { - log.Printf("Stat TEE_CUSTOMIZED_OPA error: %v", err) - return false - } - return true -} - type PodOption func(*corev1.Pod) func WithRestartPolicy(restartPolicy corev1.RestartPolicy) PodOption { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 4f491890c..03ad12af3 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -573,6 +573,8 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs } +// DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaning opa in kbs +// as test cases might be run in parallel func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { log.Info("Do test kbs key release") @@ -596,6 +598,8 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run() } +// DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaning opa in kbs +// as test cases might be run in parallel func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) { log.Info("Do test kbs key release failure case") diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index d6198ac50..6d846def8 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -97,26 +97,14 @@ func TestLibvirtPodsMTLSCommunication(t *testing.T) { DoTestPodsMTLSCommunication(t, testEnv, assert) } -func TestLibvirtKbsKeyReleaseWithDefaultOpa(t *testing.T) { +func TestLibvirtKbsKeyRelease(t *testing.T) { if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } - if !isTestWithCustomizedOpa() { - t.Skip("Skipping TestLibvirtKbsKeyReleaseWithCustomizedOpa as default opa is used") - } - assert := LibvirtAssert{} - t.Parallel() - DoTestKbsKeyRelease(t, testEnv, assert) -} - -func TestLibvirtKbsKeyReleaseWithCustomizedOpa(t *testing.T) { - if !isTestWithKbs() { - t.Skip("Skipping kbs related test as kbs is not deployed") - } - if isTestWithCustomizedOpa() { - t.Skip("Skipping TestLibvirtKbsKeyReleaseWithDefaultOpa as customized opa is used") - } + _ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego") assert := LibvirtAssert{} t.Parallel() DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + _ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego") + DoTestKbsKeyRelease(t, testEnv, assert) } diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 7a3515aa5..1d05035d3 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -98,10 +98,6 @@ func TestMain(m *testing.M) { // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA shouldDeployKbs := isTestWithKbs() - // The TEE_CUSTOMIZED_OPA is an optional variable which specifies the opa file path - // such as: $HOME/trustee/kbs/sample_policies/allow_all.rego. - customizedOpaFile := os.Getenv("TEE_CUSTOMIZED_OPA") - if !shouldProvisionCluster { // Look for a suitable kubeconfig file in the sequence: --kubeconfig flag, // or KUBECONFIG variable, or $HOME/.kube/config. @@ -153,17 +149,8 @@ func TestMain(m *testing.M) { return ctx, err } - kbsparams = "cc_kbc::http://" + kbsEndpoint + kbsparams = "cc_kbc::" + kbsEndpoint log.Infof("KBS PARAMS: %s", kbsparams) - if customizedOpaFile != "" { - log.Info("Enable customized opa file in KBS service.") - if _, err := os.Stat(customizedOpaFile); err != nil { - return ctx, err - } - if err = keyBrokerService.EnableKbsCustomizedPolicy("http://"+kbsEndpoint, customizedOpaFile); err != nil { - return ctx, err - } - } } if podvmImage != "" { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index acf0602f6..1fd970fac 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -63,6 +63,7 @@ type NewInstallOverlayFunc func(installDir, provider string) (InstallOverlay, er type KeyBrokerService struct { installOverlay InstallOverlay // Pointer to the kustomize overlay + endpoint string // KBS Service endpoint, such as: http://NodeIP:Port } var NewInstallOverlayFunctions = make(map[string]NewInstallOverlayFunc) @@ -190,6 +191,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { return &KeyBrokerService{ installOverlay: overlay, + endpoint: "", }, nil } @@ -373,19 +375,20 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf return "", err } - serviceEndpoint := fmt.Sprintf("%s:%d", nodeIP, nodePort) - return serviceEndpoint, nil + p.endpoint = fmt.Sprintf("http://%s:%d", nodeIP, nodePort) + return p.endpoint, nil } } return "", fmt.Errorf("Service %s not found", serviceName) } -func (p *KeyBrokerService) EnableKbsCustomizedPolicy(kbsEndpoint string, customizedOpaFile string) error { - log.Info("EnableKbsCustomizedPolicy") +func (p *KeyBrokerService) EnableKbsCustomizedPolicy(customizedOpaFile string) error { kbsClientDir := filepath.Join(TRUSTEE_REPO_PATH, "target/release") privateKey := "../../kbs/config/kubernetes/base/kbs.key" - cmd := exec.Command("./kbs-client", "--url", kbsEndpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", customizedOpaFile) + policyFile := filepath.Join("../../kbs/sample_policies", customizedOpaFile) + log.Info("EnableKbsCustomizedPolicy: ", policyFile) + cmd := exec.Command("./kbs-client", "--url", p.endpoint, "config", "--auth-private-key", privateKey, "set-resource-policy", "--policy-file", policyFile) cmd.Dir = kbsClientDir cmd.Env = os.Environ() stdoutStderr, err := cmd.CombinedOutput() diff --git a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go index b3a39dd18..4f862b1bf 100644 --- a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go +++ b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go @@ -109,7 +109,7 @@ func main() { log.Fatal(err) } - kbsparams := "cc_kbc::http://" + kbsEndpoint + kbsparams := "cc_kbc::" + kbsEndpoint log.Infof("KBS PARAMS: %s", kbsparams) props = provisioner.GetProperties(context.TODO(), cfg) From 8b3aafc2a0278339cc4a29b5b239186c650ac82e Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Sat, 11 May 2024 10:08:13 +0800 Subject: [PATCH 4/7] libvirt: enable attestation test cases and update readme for AA_KBC Fixes: #1825 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/test/e2e/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 0bc9bfdcd..3eb147022 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -64,6 +64,14 @@ popd popd ``` +We need build and use the PodVM image with `AA_KBC=cc_kbc` enabled, for example: +``` +pushd ${cloud-api-adaptor} +AA_KBC=cc_kbc make podvm-builder podvm-binaries podvm-image +popd +``` +Then extract the PodVM image and use it following [extracting-the-qcow2-image](../../podvm/README.md#extracting-the-qcow2-image) + To deploy the KBS service and test attestation related cases, set `export DEPLOY_KBS=yes`. ## Provision file specifics From 0a103cc8bb246d45cfe138efbd7d90c1767ca39b Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Mon, 13 May 2024 16:50:46 +0800 Subject: [PATCH 5/7] libvirt: change AA_KBC default to cc_kbc Fixes: #1825 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/podvm/Makefile.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloud-api-adaptor/podvm/Makefile.inc b/src/cloud-api-adaptor/podvm/Makefile.inc index c58245d89..430805db7 100644 --- a/src/cloud-api-adaptor/podvm/Makefile.inc +++ b/src/cloud-api-adaptor/podvm/Makefile.inc @@ -23,7 +23,7 @@ ARCH := $(or $(ARCH),$(HOST_ARCH)) # Normalise x86_64 / amd64 for input ARCH ARCH := $(subst amd64,x86_64,$(ARCH)) DEB_ARCH := $(subst x86_64,amd64,$(ARCH)) -AA_KBC ?= offline_fs_kbc +AA_KBC ?= cc_kbc KBC_URI ?= null LIBC ?= $(if $(filter $(ARCH),s390x ppc64le),gnu,musl) RUST_ARCH ?= $(subst ppc64le,powerpc64le,$(ARCH)) From 42e5d7edbbf5844384408097f6db9eac1681904f Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Mon, 13 May 2024 22:39:29 +0800 Subject: [PATCH 6/7] libvirt: update readme Fixes: #1825 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/test/e2e/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 3eb147022..919f6e5d0 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -53,7 +53,7 @@ To use existing cluster which have already installed Cloud API Adaptor, you shou We need artificats from trustee when do attestation test, to prepare trustee, do as following. ``` -pushd ${cloud-api-adaptor}/test/e2e +pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test/e2e git clone https://github.com/confidential-containers/trustee.git pushd trustee git checkout $(../../../hack/yq-shim.sh '.git.kbs.reference' ../../../versions.yaml) @@ -72,7 +72,12 @@ popd ``` Then extract the PodVM image and use it following [extracting-the-qcow2-image](../../podvm/README.md#extracting-the-qcow2-image) -To deploy the KBS service and test attestation related cases, set `export DEPLOY_KBS=yes`. +To deploy the KBS service and test attestation related cases, export following variables like: +``` +export DEPLOY_KBS=yes +export KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml) +export KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml) +```` ## Provision file specifics From 512f8581da2b96c10f644efac14458f4dcf1e7a5 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Tue, 14 May 2024 09:10:08 +0800 Subject: [PATCH 7/7] libvirt: enable e2e test and rollback TRUSTEE_REPO_PATH Fixes: #1825 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/test/e2e/README.md | 4 ++-- src/cloud-api-adaptor/test/provisioner/provision.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 919f6e5d0..d5d4c45fe 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -53,10 +53,10 @@ To use existing cluster which have already installed Cloud API Adaptor, you shou We need artificats from trustee when do attestation test, to prepare trustee, do as following. ``` -pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test/e2e +pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test git clone https://github.com/confidential-containers/trustee.git pushd trustee -git checkout $(../../../hack/yq-shim.sh '.git.kbs.reference' ../../../versions.yaml) +git checkout $(../../hack/yq-shim.sh '.git.kbs.reference' ../../versions.yaml) pushd kbs make CLI_FEATURES=sample_only cli popd diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 1fd970fac..be5dcd06f 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -82,7 +82,7 @@ type InstallOverlay interface { const PodWaitTimeout = time.Second * 30 // trustee repo related base path -const TRUSTEE_REPO_PATH = "trustee" +const TRUSTEE_REPO_PATH = "../trustee" func saveToFile(filename string, content []byte) error { // Save contents to file