From 8ad754ef735c2e06b9c04e7079deb7b41e7bd11f Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Tue, 14 Nov 2023 14:13:50 +0530 Subject: [PATCH] Provisioner: Add support to fetch kbs service ip Fixes: #1471 Signed-off-by: Kartik Joshi --- test/e2e/main_test.go | 26 ++++--- .../azure/provision_azure.properties | 2 +- test/provisioner/provision.go | 78 +++++++++++-------- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index c04e87b3c6..9095990973 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -18,9 +18,9 @@ import ( ) var ( - testEnv env.Environment - cloudProvider string - provisioner pv.CloudProvisioner + testEnv env.Environment + cloudProvider string + provisioner pv.CloudProvisioner keyBrokerService *pv.KeyBrokerService ) @@ -96,9 +96,9 @@ func TestMain(m *testing.M) { } // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA - shouldDeployKbs := false - if os.Getenv("DEPLOY_KBS") == "yes" { - shouldDeployKbs = true + shouldDeployKbs := true + if os.Getenv("DEPLOY_KBS") != "yes" { + shouldDeployKbs = false } if !shouldProvisionCluster { @@ -117,7 +117,6 @@ func TestMain(m *testing.M) { log.Info("Do setup") var err error // Get properties - props := provisioner.GetProperties(ctx, cfg) if shouldProvisionCluster { log.Info("Cluster provisioning") @@ -130,6 +129,8 @@ func TestMain(m *testing.M) { } } + props := provisioner.GetProperties(ctx, cfg) + var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { @@ -144,12 +145,12 @@ func TestMain(m *testing.M) { return ctx, err } var kbsPodIP string - if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil { + if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { return ctx, err } - kbsparams := "cc_kbc::http:" + kbsPodIP + ":8080" - props["AA_KBC_PARAMS"] = kbsparams + kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080" + log.Infof("KBS PARAMS%s:", kbsparams) } if podvmImage != "" { @@ -165,8 +166,11 @@ func TestMain(m *testing.M) { if cloudAPIAdaptor, err = pv.NewCloudAPIAdaptor(cloudProvider, relativeInstallDirectory); err != nil { return ctx, err } + + props = provisioner.GetProperties(ctx, cfg) + props["AA_KBC_PARAMS"] = kbsparams log.Info("Deploy the Cloud API Adaptor") - if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil { + if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil { return ctx, err } } diff --git a/test/provisioner/azure/provision_azure.properties b/test/provisioner/azure/provision_azure.properties index 3dfc7bc1aa..0b19cff4c5 100644 --- a/test/provisioner/azure/provision_azure.properties +++ b/test/provisioner/azure/provision_azure.properties @@ -13,5 +13,5 @@ SSH_USERNAME="" AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" -KBS_IMAGE="quay.io/karikjoshi21/kbs/coco-as-21705eb" +KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs" KBS_IMAGE_TAG="latest" diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index e989d1e1db..8d118cc2e9 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -103,7 +103,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Create secret content := []byte("This is my cluster name: " + clusterName) - filePath := "kbs/config/kubernetes/overlays/key.bin" + filePath := "kbs/kbs/config/kubernetes/overlays/key.bin" // Create the file. file, err := os.Create(filePath) if err != nil { @@ -129,16 +129,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } fmt.Println(k8sCnfDir) - keyFile := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/overlays/key.bin") + keyFile := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/overlays/key.bin") if _, err := os.Stat(keyFile); os.IsNotExist(err) { err = fmt.Errorf("key.bin file does not exist") log.Errorf("%v", err) return nil, err } - kbsCert := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.pem") + kbsCert := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.pem") if _, err := os.Stat(kbsCert); os.IsNotExist(err) { - kbsKey := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.key") + kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key") keyOutputFile, err := os.Create(kbsKey) if err != nil { err = fmt.Errorf("Error creating key file: %w\n", err) @@ -224,7 +224,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { log.Info("Creating kbs install overlay") - overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "config/kubernetes/base")) + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base")) if err != nil { return nil, err } @@ -257,7 +257,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro return nil } -func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) { +func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) (string, error) { client, err := cfg.NewClient() if err != nil { return "", err @@ -275,29 +275,30 @@ func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) resources := client.Resources(namespace) - podList := &corev1.PodList{} - err = resources.List(context.TODO(), podList) - if err != nil { - err = fmt.Errorf("Error listing pods: %w\n", err) - log.Errorf("%v", err) - return "", err - } - - var matchingPod *corev1.Pod - for i := range podList.Items { - pod := &podList.Items[i] - if pod.Labels["app"] == deploymentName { - matchingPod = pod - break - } - } - - if matchingPod == nil { - return "", fmt.Errorf("No pod with label selector found") - } - - fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP) - return matchingPod.Status.PodIP, nil + // Get the service associated with the deployment + serviceList := &corev1.ServiceList{} + err = resources.List(context.TODO(), serviceList) + if err != nil { + err = fmt.Errorf("Error listing services: %w\n", err) + log.Errorf("%v", err) + return "", err + } + + var matchingService *corev1.Service + for i := range serviceList.Items { + service := &serviceList.Items[i] + if service.Name == deploymentName { + matchingService = service + break + } + } + + if matchingService == nil { + return "", fmt.Errorf("No service with label selector found") + } + + fmt.Printf("Service IP: %s\n", matchingService.Spec.ClusterIP) + return matchingService.Spec.ClusterIP, nil } func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { @@ -313,7 +314,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } - newDirectory := "kbs/config/kubernetes/overlays" + newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { err = fmt.Errorf("Error changing the working directory: %w\n", err) @@ -321,10 +322,21 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } + // Replace this to use install overlay + cmd := exec.Command("kubectl", "apply", "-k", "overlays") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + + /* log.Info("Install Kbs") if err := p.installOverlay.Apply(ctx, cfg); err != nil { return err } + */ // Return to the original working directory. err = os.Chdir(originalDir) @@ -347,7 +359,7 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro } // Remove kbs deployment - newDirectory := "kbs/config/kubernetes/overlays" + newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { err = fmt.Errorf("Error changing the working directory: %w\n", err) @@ -355,7 +367,7 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro return err } - log.Info("Install Kbs") + log.Info("Delete Kbs deployment") if err := p.installOverlay.Delete(ctx, cfg); err != nil { return err } @@ -582,7 +594,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string) for _, o := range metaList { obj, _ := o.(k8s.Object) fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName()) - if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*6)); err != nil { + if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil { return err } fmt.Printf("pod '%s' is Ready\n", obj.GetName())