From 119fb750ffd126b76a9f3aa7586b62ae2e32085e Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Tue, 24 Sep 2024 17:00:26 +0100 Subject: [PATCH 1/4] test/e2e: Update KBS key path - In `NewKeyBrokerService` we create a default secret in `reponame/workload_key/key.bin` as the KBS needs at least one secret to start - In `SetSampleSecretKey` we are then resetting a resource in the same path, which causes an error ``` Set secret failed: write local fs ``` I guess as the kbs process owns that directory. - Update the code to set a new secret for each test and use a different path to prevent this clash Signed-off-by: stevenhorsman --- src/cloud-api-adaptor/test/e2e/azure_test.go | 6 ++- src/cloud-api-adaptor/test/e2e/common.go | 1 - .../test/e2e/common_suite.go | 27 +++++++------ src/cloud-api-adaptor/test/e2e/docker_test.go | 9 +++-- .../test/e2e/libvirt_test.go | 21 ++++++---- .../test/provisioner/trustee_kbs.go | 39 ++++++++----------- 6 files changed, 55 insertions(+), 48 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index 958536bd4..d3bb53e44 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -12,6 +12,7 @@ import ( "testing" _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/azure" + "sigs.k8s.io/e2e-framework/pkg/envconf" ) func TestDeletePodAzure(t *testing.T) { @@ -131,7 +132,10 @@ func TestKbsKeyRelease(t *testing.T) { } t.Parallel() kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() - DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) + testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) + resourcePath := "caa/workload_key/test_key.bin" + err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } func TestRemoteAttestation(t *testing.T) { diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index 29a8cf49f..77143911d 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -29,7 +29,6 @@ import ( const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180 const DEFAULT_AUTH_SECRET = "auth-json-secret-default" -const KBS_SECRET = "reponame/workload_key/key.bin" var testInitdata string = `algorithm = "sha384" version = "0.1.0" diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index f6227e4eb..be7b57a5a 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -547,9 +547,9 @@ func DoTestImageDecryption(t *testing.T, e env.Environment, assert CloudAssert, NewTestCase(t, e, "TestImageDecryption", assert, "Encrypted image layers have been decrypted").WithPod(pod).WithDeleteAssertion(&duration).Run() } -func DoTestSealedSecret(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { +func DoTestSealedSecret(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string, resourcePath, expectedSecret string) { key := "MY_SECRET" - value := CreateSealedSecretValue("kbs:///" + KBS_SECRET) + value := CreateSealedSecretValue("kbs:///" + resourcePath) podName := "sealed-secret" imageName := getBusyboxTestImage(t) env := []v1.EnvVar{{Name: key, Value: value}} @@ -557,25 +557,24 @@ func DoTestSealedSecret(t *testing.T, e env.Environment, assert CloudAssert, kbs pod := NewPod(E2eNamespace, podName, podName, imageName, WithEnvironmentVariables(env), WithInitdata(kbsEndpoint), WithCommand(cmd)) - expectedPodLogString := "This is my" - NewTestCase(t, e, "TestSealedSecret", assert, "Unsealed secret has been set to ENV").WithPod(pod).WithExpectedPodLogString(expectedPodLogString).Run() + NewTestCase(t, e, "TestSealedSecret", assert, "Unsealed secret has been set to ENV").WithPod(pod).WithExpectedPodLogString(expectedSecret).Run() } // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs // as test cases might be run in parallel -func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { +func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint, resourcePath, expectedSecret string) { t.Log("Do test kbs key release") pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "kbs-key-release", kbsEndpoint).GetPodOrFatal(t) testCommands := []TestCommand{ { - Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + KBS_SECRET}, + Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + resourcePath}, ContainerName: pod.Spec.Containers[0].Name, TestCommandStdoutFn: func(stdout bytes.Buffer) bool { - if strings.Contains(stdout.String(), "This is my cluster name") { - t.Logf("Success to get key.bin: %s", stdout.String()) + if strings.Contains(stdout.String(), expectedSecret) { + t.Logf("Success to get secret key: %s", stdout.String()) return true } else { - t.Errorf("Failed to access key.bin: %s", stdout.String()) + t.Errorf("Failed to access secret key: %s", stdout.String()) return false } }, @@ -587,12 +586,12 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kb // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs // as test cases might be run in parallel -func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { +func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint, resourcePath, expectedSecret string) { t.Log("Do test kbs key release failure case") pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "kbs-failure", kbsEndpoint).GetPodOrFatal(t) testCommands := []TestCommand{ { - Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + KBS_SECRET}, + Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + resourcePath}, ContainerName: pod.Spec.Containers[0].Name, TestErrorFn: func(err error) bool { if strings.Contains(err.Error(), "command terminated with exit code 1") { @@ -603,11 +602,11 @@ func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert Cloud } }, TestCommandStdoutFn: func(stdout bytes.Buffer) bool { - if strings.Contains(stdout.String(), "This is my cluster name") { - t.Errorf("FAIL as successed to get key.bin: %s", stdout.String()) + if strings.Contains(stdout.String(), expectedSecret) { + t.Errorf("FAIL as succeed to get secret key: %s", stdout.String()) return false } else { - t.Logf("PASS as failed to access key.bin: %s", stdout.String()) + t.Logf("PASS as failed to access secret key: %s", stdout.String()) return true } }, diff --git a/src/cloud-api-adaptor/test/e2e/docker_test.go b/src/cloud-api-adaptor/test/e2e/docker_test.go index 4c772267d..f43ab86ec 100644 --- a/src/cloud-api-adaptor/test/e2e/docker_test.go +++ b/src/cloud-api-adaptor/test/e2e/docker_test.go @@ -10,6 +10,7 @@ import ( "testing" _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/docker" + "sigs.k8s.io/e2e-framework/pkg/envconf" ) func TestDockerCreateSimplePod(t *testing.T) { @@ -102,14 +103,16 @@ func TestDockerKbsKeyRelease(t *testing.T) { if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } - keyBrokerService.SetSampleSecretKey() + testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) + resourcePath := "caa/workload_key/test_key.bin" + keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) keyBrokerService.EnableKbsCustomizedResourcePolicy("deny_all.rego") kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() assert := DockerAssert{} t.Parallel() - DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") - DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } func TestDockerCreatePeerPodWithAuthenticatedImageWithoutCredentials(t *testing.T) { diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index e8503fae9..682669bad 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt" + "sigs.k8s.io/e2e-framework/pkg/envconf" ) func TestLibvirtCreateSimplePod(t *testing.T) { @@ -137,37 +138,43 @@ func TestLibvirtSealedSecret(t *testing.T) { if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } - _ = keyBrokerService.SetSampleSecretKey() + + testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) + resourcePath := "caa/workload_key/test_key.bin" + _ = keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) _ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() assert := LibvirtAssert{} - DoTestSealedSecret(t, testEnv, assert, kbsEndpoint) + DoTestSealedSecret(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } func TestLibvirtKbsKeyRelease(t *testing.T) { if !isTestWithKbs() { t.Skip("Skipping kbs related test as kbs is not deployed") } - _ = keyBrokerService.SetSampleSecretKey() + + testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) + resourcePath := "caa/workload_key/test_key.bin" + _ = keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) _ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego") kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() assert := LibvirtAssert{} t.Parallel() - DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) if isTestWithKbsIBMSE() { t.Log("KBS with ibmse cases") // the allow_*_.rego file is created by follow document // https://github.com/confidential-containers/trustee/blob/main/deps/verifier/src/se/README.md#set-attestation-policy _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego") - DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego") - DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } else { t.Log("KBS normal cases") _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") - DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } } diff --git a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go index 18d91e7a9..c78e04b6a 100644 --- a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go +++ b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go @@ -12,6 +12,7 @@ import ( "fmt" "os" "os/exec" + "path" "path/filepath" "strings" "time" @@ -68,20 +69,9 @@ func NewKeyBrokerService(clusterName string, cfg *envconf.Config) (*KeyBrokerSer } filePath := filepath.Join(getKbsKubernetesFilePath(), overlaysPath, "key.bin") - // Create the file. - file, err := os.Create(filePath) - if err != nil { - err = fmt.Errorf("creating file: %w\n", err) - log.Errorf("%v", err) - return nil, err - } - defer file.Close() - // Write the content to the file. - err = saveToFile(filePath, content) + err = os.WriteFile(filePath, content, 0644) if err != nil { - err = fmt.Errorf("writing to the file: %w\n", err) - log.Errorf("%v", err) return nil, err } @@ -502,13 +492,27 @@ func (p *KeyBrokerService) setSecretKey(resource string, path string) error { cmd.Dir = trusteeRepoPath cmd.Env = os.Environ() stdoutStderr, err := cmd.CombinedOutput() - log.Tracef("%v, output: %s", cmd, stdoutStderr) + log.Tracef("%v, status: %v, output: %s", cmd, err, stdoutStderr) if err != nil { return err } return nil } +func (p *KeyBrokerService) SetSecret(resourcePath string, secret []byte) error { + tempDir, _ := os.MkdirTemp("", "kbs_resource_files") + + defer os.RemoveAll(tempDir) + + var secretFilePath = filepath.Join(tempDir, path.Base(resourcePath)) + err := os.WriteFile(secretFilePath, secret, 0644) + if err != nil { + return err + } + + return p.setSecretKey(resourcePath, secretFilePath) +} + func (p *KeyBrokerService) SetImageDecryptionKey(keyID string, key []byte) error { if len(key) != 32 { return fmt.Errorf("image decryption key must be 32 bytes") @@ -525,15 +529,6 @@ func (p *KeyBrokerService) SetImageDecryptionKey(keyID string, key []byte) error return p.setSecretKey(keyID, path.Name()) } -func (p *KeyBrokerService) SetSampleSecretKey() error { - overlaysPath, err := getOverlaysPath() - if err != nil { - return err - } - path := filepath.Join(getKbsKubernetesFilePath(), overlaysPath, "key.bin") - return p.setSecretKey("reponame/workload_key/key.bin", path) -} - func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { log.Info("Customize the overlay yaml file") if err := p.installOverlay.Edit(ctx, cfg, props); err != nil { From 483828640fe2d1a6d9ee5838017b2bc453cfbbc3 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Tue, 24 Sep 2024 17:02:11 +0100 Subject: [PATCH 2/4] test/e2e: azure: Refactor out TestTrusteeOperatorKeyReleaseForSpecificKey Now the `DoTestKbsKeyRelease` test can have customised secret and resource path, we can re-use it for the trustee test Signed-off-by: stevenhorsman --- src/cloud-api-adaptor/test/e2e/azure_test.go | 2 +- .../test/e2e/common_suite.go | 23 ------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index d3bb53e44..bca57f7ee 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -157,7 +157,7 @@ func TestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T) { } t.Parallel() kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() - DoTestTrusteeOperatorKeyReleaseForSpecificKey(t, testEnv, assert, kbsEndpoint) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, "default/kbsres1/key1", "res1val1") } func TestAzureImageDecryption(t *testing.T) { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index be7b57a5a..13a4dce61 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -616,29 +616,6 @@ func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert Cloud NewTestCase(t, e, "DoTestKbsKeyReleaseForFailure", assert, "Kbs key release is failed").WithPod(pod).WithTestCommands(testCommands).Run() } -// Test to check for specific key value from Trustee Operator Deployment -func DoTestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { - t.Log("Do test Trustee operator key release for specific key") - pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "op-key-release", kbsEndpoint).GetPodOrFatal(t) - testCommands := []TestCommand{ - { - Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/default/kbsres1/key1"}, - ContainerName: pod.Spec.Containers[0].Name, - TestCommandStdoutFn: func(stdout bytes.Buffer) bool { - if strings.Contains(stdout.String(), "res1val1") { - t.Logf("Success to get key %s", stdout.String()) - return true - } else { - t.Errorf("Failed to access key: %s", stdout.String()) - return false - } - }, - }, - } - - NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release from Trustee Operator is successful").WithPod(pod).WithTestCommands(testCommands).Run() -} - func DoTestRestrictivePolicyBlocksExec(t *testing.T, e env.Environment, assert CloudAssert) { allowAllExceptExecPolicyFilePath := "fixtures/policies/allow-all-except-exec-process.rego" podName := "policy-exec-rejected" From 212e0501ce8347b4dacea00704166d927ccfbaaa Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Tue, 24 Sep 2024 17:12:39 +0100 Subject: [PATCH 3/4] test/e2e: Test should fail if we error during set-up We have had an issue where the secret key setting wasn't working and throwing errors and we just ignored it. To help with debugging we should be responding to errors rather than just ignoring them. Signed-off-by: stevenhorsman --- src/cloud-api-adaptor/test/e2e/azure_test.go | 14 ++++- src/cloud-api-adaptor/test/e2e/docker_test.go | 20 +++++-- .../test/e2e/libvirt_test.go | 55 +++++++++++++++---- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index bca57f7ee..fdf0b1f26 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -135,6 +135,9 @@ func TestKbsKeyRelease(t *testing.T) { testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) resourcePath := "caa/workload_key/test_key.bin" err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) + if err != nil { + t.Fatalf("SetSecret failed with: %v", err) + } DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } @@ -146,7 +149,11 @@ func TestRemoteAttestation(t *testing.T) { } else if keyBrokerService == nil { t.Skip("Skipping because KBS config is missing") } else { - kbsEndpoint, _ = keyBrokerService.GetCachedKbsEndpoint() + var err error + kbsEndpoint, err = keyBrokerService.GetCachedKbsEndpoint() + if err != nil { + t.Fatalf("GetCachedKbsEndpoint failed with: %v", err) + } } DoTestRemoteAttestation(t, testEnv, assert, kbsEndpoint) } @@ -156,7 +163,10 @@ func TestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T) { t.Skip("Skipping kbs related test as Trustee Operator is not deployed") } t.Parallel() - kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() + kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint() + if err != nil { + t.Fatalf("GetCachedKbsEndpoint failed with: %v", err) + } DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, "default/kbsres1/key1", "res1val1") } diff --git a/src/cloud-api-adaptor/test/e2e/docker_test.go b/src/cloud-api-adaptor/test/e2e/docker_test.go index f43ab86ec..25124f5b0 100644 --- a/src/cloud-api-adaptor/test/e2e/docker_test.go +++ b/src/cloud-api-adaptor/test/e2e/docker_test.go @@ -105,13 +105,25 @@ func TestDockerKbsKeyRelease(t *testing.T) { } testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) resourcePath := "caa/workload_key/test_key.bin" - keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) - keyBrokerService.EnableKbsCustomizedResourcePolicy("deny_all.rego") - kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() + err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) + if err != nil { + t.Fatalf("SetSecret failed with: %v", err) + } + err = keyBrokerService.EnableKbsCustomizedResourcePolicy("deny_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err) + } + kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint() + if err != nil { + t.Fatalf("GetCachedKbsEndpoint failed with: %v", err) + } assert := DockerAssert{} t.Parallel() DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) - keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") + err = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err) + } DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 682669bad..d4b64d217 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -141,10 +141,22 @@ func TestLibvirtSealedSecret(t *testing.T) { testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) resourcePath := "caa/workload_key/test_key.bin" - _ = keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) - _ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") - _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") - kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() + err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) + if err != nil { + t.Fatalf("SetSecret failed with: %v", err) + } + err = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err) + } + err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err) + } + kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint() + if err != nil { + t.Fatalf("GetCachedKbsEndpoint failed with: %v", err) + } assert := LibvirtAssert{} DoTestSealedSecret(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } @@ -156,10 +168,22 @@ func TestLibvirtKbsKeyRelease(t *testing.T) { testSecret := envconf.RandomName("coco-pp-e2e-secret", 25) resourcePath := "caa/workload_key/test_key.bin" - _ = keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) - _ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") - _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego") - kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() + err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret)) + if err != nil { + t.Fatalf("SetSecret failed with: %v", err) + } + err = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err) + } + err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err) + } + kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint() + if err != nil { + t.Fatalf("GetCachedKbsEndpoint failed with: %v", err) + } assert := LibvirtAssert{} t.Parallel() DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) @@ -167,13 +191,22 @@ func TestLibvirtKbsKeyRelease(t *testing.T) { t.Log("KBS with ibmse cases") // the allow_*_.rego file is created by follow document // https://github.com/confidential-containers/trustee/blob/main/deps/verifier/src/se/README.md#set-attestation-policy - _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego") + err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err) + } DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) - _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego") + err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err) + } DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } else { t.Log("KBS normal cases") - _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") + err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") + if err != nil { + t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err) + } DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret) } } From 6356d3410fb9ed9a72994940b5d9e3e7d5be9211 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Wed, 25 Sep 2024 10:25:47 +0100 Subject: [PATCH 4/4] tests/e2e: Minor tidy-up and logging - Trace out the test command being run for debug - Remove unnecessary double if check Signed-off-by: stevenhorsman --- src/cloud-api-adaptor/test/e2e/assessment_helpers.go | 9 +++++---- src/cloud-api-adaptor/test/e2e/assessment_runner.go | 8 +++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go index 7de1951d8..10976f332 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go @@ -472,6 +472,7 @@ func AssessPodTestCommands(ctx context.Context, client klient.Client, pod *v1.Po return "Failed to list pod", err } for _, testCommand := range testCommands { + log.Tracef("Running test command: %v", testCommand) var stdout, stderr bytes.Buffer for _, podItem := range podlist.Items { if podItem.ObjectMeta.Name == pod.Name { @@ -480,24 +481,24 @@ func AssessPodTestCommands(ctx context.Context, client klient.Client, pod *v1.Po if err := client.Resources(pod.Namespace).ExecInPod(ctx, pod.Namespace, pod.Name, testCommand.ContainerName, testCommand.Command, &stdout, &stderr); err != nil { if testCommand.TestErrorFn != nil { if !testCommand.TestErrorFn(err) { - return err.Error(), fmt.Errorf("Command %v running in container %s produced unexpected output on error: %s", testCommand.Command, testCommand.ContainerName, err.Error()) + return err.Error(), fmt.Errorf("command %v running in container %s produced unexpected output on error: %s, stderr: %s", testCommand.Command, testCommand.ContainerName, err.Error(), stderr.String()) } } else { - return err.Error(), err + return err.Error(), fmt.Errorf("command %v running in container %s produced unexpected output on error: %s, stderr: %s", testCommand.Command, testCommand.ContainerName, err.Error(), stderr.String()) } } else if testCommand.TestErrorFn != nil { return "", fmt.Errorf("We expected an error from Pod %s, but it was not found", pod.Name) } if testCommand.TestCommandStderrFn != nil { if !testCommand.TestCommandStderrFn(stderr) { - return stderr.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stderr: %s", testCommand.Command, testCommand.ContainerName, stderr.String()) + return stderr.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stderr: %s, stdout: %s", testCommand.Command, testCommand.ContainerName, stderr.String(), stdout.String()) } else { return stderr.String(), nil } } if testCommand.TestCommandStdoutFn != nil { if !testCommand.TestCommandStdoutFn(stdout) { - return stdout.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stdout: %s", testCommand.Command, testCommand.ContainerName, stdout.String()) + return stdout.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stdout: %s, stderr: %s", testCommand.Command, testCommand.ContainerName, stdout.String(), stderr.String()) } else { return stdout.String(), nil } diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go index 15788c6b0..15e75f723 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_runner.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_runner.go @@ -393,11 +393,9 @@ func (tc *TestCase) Run() { if tc.podState == v1.PodRunning { if len(tc.testCommands) > 0 { - if len(tc.testCommands) > 0 { - logString, err := AssessPodTestCommands(ctx, client, tc.pod, tc.testCommands) - if err != nil { - t.Errorf("AssessPodTestCommands failed, with output: %s and error: %v", logString, err) - } + logString, err := AssessPodTestCommands(ctx, client, tc.pod, tc.testCommands) + if err != nil { + t.Errorf("AssessPodTestCommands failed, with output: %s and error: %v", logString, err) } }