From b5c4e5f190e4c72389189a1358acd7c4f24e3684 Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Tue, 14 Nov 2023 14:13:50 +0530 Subject: [PATCH] Provisioner: Add support to fetch kbs service ip Fixes: #1471 Signed-off-by: Kartik Joshi --- test/e2e/main_test.go | 26 ++-- .../azure/provision_azure.properties | 2 +- test/provisioner/provision.go | 118 +++++++++++++----- 3 files changed, 101 insertions(+), 45 deletions(-) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index c04e87b3c6..5dc326c228 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -18,9 +18,9 @@ import ( ) var ( - testEnv env.Environment - cloudProvider string - provisioner pv.CloudProvisioner + testEnv env.Environment + cloudProvider string + provisioner pv.CloudProvisioner keyBrokerService *pv.KeyBrokerService ) @@ -96,9 +96,9 @@ func TestMain(m *testing.M) { } // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA - shouldDeployKbs := false - if os.Getenv("DEPLOY_KBS") == "yes" { - shouldDeployKbs = true + shouldDeployKbs := true + if os.Getenv("DEPLOY_KBS") != "yes" { + shouldDeployKbs = false } if !shouldProvisionCluster { @@ -117,7 +117,6 @@ func TestMain(m *testing.M) { log.Info("Do setup") var err error // Get properties - props := provisioner.GetProperties(ctx, cfg) if shouldProvisionCluster { log.Info("Cluster provisioning") @@ -130,6 +129,8 @@ func TestMain(m *testing.M) { } } + props := provisioner.GetProperties(ctx, cfg) + var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { @@ -144,12 +145,12 @@ func TestMain(m *testing.M) { return ctx, err } var kbsPodIP string - if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil { + if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { return ctx, err } - kbsparams := "cc_kbc::http:" + kbsPodIP + ":8080" - props["AA_KBC_PARAMS"] = kbsparams + kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080" + log.Infof("KBS PARAMS%s:", kbsparams) } if podvmImage != "" { @@ -165,8 +166,11 @@ func TestMain(m *testing.M) { if cloudAPIAdaptor, err = pv.NewCloudAPIAdaptor(cloudProvider, relativeInstallDirectory); err != nil { return ctx, err } + + props = provisioner.GetProperties(ctx, cfg) + props["AA_KBC_PARAMS"] = kbsparams log.Info("Deploy the Cloud API Adaptor") - if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil { + if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil { return ctx, err } } diff --git a/test/provisioner/azure/provision_azure.properties b/test/provisioner/azure/provision_azure.properties index 3dfc7bc1aa..0b19cff4c5 100644 --- a/test/provisioner/azure/provision_azure.properties +++ b/test/provisioner/azure/provision_azure.properties @@ -13,5 +13,5 @@ SSH_USERNAME="" AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" -KBS_IMAGE="quay.io/karikjoshi21/kbs/coco-as-21705eb" +KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs" KBS_IMAGE_TAG="latest" diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index e989d1e1db..5420b6fd96 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -5,6 +5,10 @@ package provisioner import ( "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" "fmt" "io" "os" @@ -91,6 +95,15 @@ func runCommand(command string, stdout io.Writer, stderr io.Writer, args ...stri return nil } +func saveToFile(filename string, content []byte) error { + // Save contents to file + err := os.WriteFile(filename, content, 0644) + if err != nil { + return fmt.Errorf("error writing contents to file: %w", err) + } + return nil +} + func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Clone kbs repo repoURL := "https://github.com/confidential-containers/kbs" @@ -103,7 +116,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Create secret content := []byte("This is my cluster name: " + clusterName) - filePath := "kbs/config/kubernetes/overlays/key.bin" + filePath := "kbs/kbs/config/kubernetes/overlays/key.bin" // Create the file. file, err := os.Create(filePath) if err != nil { @@ -114,7 +127,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { defer file.Close() // Write the content to the file. - _, err = file.Write(content) + err = saveToFile(filePath, content) if err != nil { err = fmt.Errorf("Error writing to the file: %w\n", err) log.Errorf("%v", err) @@ -129,16 +142,9 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } fmt.Println(k8sCnfDir) - keyFile := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/overlays/key.bin") - if _, err := os.Stat(keyFile); os.IsNotExist(err) { - err = fmt.Errorf("key.bin file does not exist") - log.Errorf("%v", err) - return nil, err - } - - kbsCert := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.pem") + kbsCert := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.pem") if _, err := os.Stat(kbsCert); os.IsNotExist(err) { - kbsKey := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.key") + kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key") keyOutputFile, err := os.Create(kbsKey) if err != nil { err = fmt.Errorf("Error creating key file: %w\n", err) @@ -147,13 +153,47 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } defer keyOutputFile.Close() - if err := runCommand("openssl", keyOutputFile, os.Stderr, "genpkey", "-algorithm", "ed25519"); err != nil { + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: privateKey, + }) + + // Save private key to file + err = saveToFile(kbsKey, privateKeyPEM) + if err != nil { + err = fmt.Errorf("Error saving private key to file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + publicKey := privateKey.Public().(ed25519.PublicKey) + publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err) + log.Errorf("%v", err) return nil, err } - if err := runCommand("openssl", os.Stdout, os.Stderr, "pkey", "-in", kbsKey, "-pubout", "-out", kbsCert); err != nil { + publicKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Bytes: publicKeyX509, + }) + + // Save public key to file + err = saveToFile(kbsCert, publicKeyPEM) + if err != nil { + err = fmt.Errorf("Error saving public key to file: %w\n", err) + log.Errorf("%v", err) return nil, err } + } overlay, err := NewKbsInstallOverlay("kbs") @@ -224,7 +264,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { log.Info("Creating kbs install overlay") - overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "config/kubernetes/base")) + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base")) if err != nil { return nil, err } @@ -257,7 +297,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro return nil } -func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) { +func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) (string, error) { client, err := cfg.NewClient() if err != nil { return "", err @@ -275,29 +315,30 @@ func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) resources := client.Resources(namespace) - podList := &corev1.PodList{} - err = resources.List(context.TODO(), podList) + // Get the service associated with the deployment + serviceList := &corev1.ServiceList{} + err = resources.List(context.TODO(), serviceList) if err != nil { - err = fmt.Errorf("Error listing pods: %w\n", err) + err = fmt.Errorf("Error listing services: %w\n", err) log.Errorf("%v", err) return "", err } - var matchingPod *corev1.Pod - for i := range podList.Items { - pod := &podList.Items[i] - if pod.Labels["app"] == deploymentName { - matchingPod = pod + var matchingService *corev1.Service + for i := range serviceList.Items { + service := &serviceList.Items[i] + if service.Name == deploymentName { + matchingService = service break } } - if matchingPod == nil { - return "", fmt.Errorf("No pod with label selector found") + if matchingService == nil { + return "", fmt.Errorf("No service with label selector found") } - fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP) - return matchingPod.Status.PodIP, nil + fmt.Printf("KBS Service IP: %s\n", matchingService.Spec.ClusterIP) + return matchingService.Spec.ClusterIP, nil } func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { @@ -313,7 +354,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } - newDirectory := "kbs/config/kubernetes/overlays" + newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { err = fmt.Errorf("Error changing the working directory: %w\n", err) @@ -321,11 +362,22 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } - log.Info("Install Kbs") - if err := p.installOverlay.Apply(ctx, cfg); err != nil { + // Replace this to use install overlay + cmd := exec.Command("kubectl", "apply", "-k", "overlays") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { return err } + /* + log.Info("Install Kbs") + if err := p.installOverlay.Apply(ctx, cfg); err != nil { + return err + } + */ + // Return to the original working directory. err = os.Chdir(originalDir) if err != nil { @@ -347,7 +399,7 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro } // Remove kbs deployment - newDirectory := "kbs/config/kubernetes/overlays" + newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { err = fmt.Errorf("Error changing the working directory: %w\n", err) @@ -355,7 +407,7 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro return err } - log.Info("Install Kbs") + log.Info("Delete Kbs deployment") if err := p.installOverlay.Delete(ctx, cfg); err != nil { return err } @@ -582,7 +634,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string) for _, o := range metaList { obj, _ := o.(k8s.Object) fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName()) - if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*6)); err != nil { + if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil { return err } fmt.Printf("pod '%s' is Ready\n", obj.GetName())