From b02ee8569eed4752724d0886ecd866680b27f5c6 Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Thu, 12 Oct 2023 17:37:49 +0530 Subject: [PATCH 1/3] Provisioner: Add support to deploy kbs Fixes: #1471 Signed-off-by: Kartik Joshi --- test/e2e/main_test.go | 31 ++++++ test/provisioner/provision.go | 180 ++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 006568b20..a3eb3c8cc 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -20,6 +20,10 @@ var ( testEnv env.Environment cloudProvider string provisioner pv.CloudProvisioner + testEnv env.Environment + cloudProvider string + provisioner pv.CloudProvisioner + keyBrokerService *pv.KeyBrokerService ) func init() { @@ -80,6 +84,9 @@ func TestMain(m *testing.M) { // the VPC images storage. podvmImage := os.Getenv("TEST_PODVM_IMAGE") + kbsImage := os.Getenv("TEST_KBS_IMAGE") + kbsImageTag := os.Getenv("TEST_KBS_IMAGE_TAG") + // The TEST_PROVISION_FILE is an optional variable which specifies the path // to the provision properties file. The file must have the format: // @@ -93,6 +100,12 @@ func TestMain(m *testing.M) { log.Fatal(err) } + // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA + shouldDeployKbs := false + if os.Getenv("DEPLOY_KBS") == "yes" { + shouldDeployKbs = true + } + if !shouldProvisionCluster { // Look for a suitable kubeconfig file in the sequence: --kubeconfig flag, // or KUBECONFIG variable, or $HOME/.kube/config. @@ -120,6 +133,18 @@ func TestMain(m *testing.M) { } } + if shouldDeployKbs { + log.Info("Deploying kbs") + + if keyBrokerService, err = pv.NewKeyBrokerService(cloudProvider); err != nil { + return ctx, err + } + + if err = keyBrokerService.Deploy(ctx, kbsImage, kbsImageTag); err != nil { + return ctx, err + } + } + if podvmImage != "" { log.Info("Podvm uploading") if err = provisioner.UploadPodvm(podvmImage, ctx, cfg); err != nil { @@ -173,6 +198,12 @@ func TestMain(m *testing.M) { } } + if shouldDeployKbs { + if err = keyBrokerService.Delete(ctx); err != nil { + return ctx, err + } + } + return ctx, nil }) diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index 0d056a299..4e220e688 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -10,6 +10,9 @@ import ( "os/exec" "path/filepath" "time" + "strings" + "path/filepath" + "io/ioutil" "github.com/BurntSushi/toml" log "github.com/sirupsen/logrus" @@ -52,6 +55,10 @@ type CloudAPIAdaptor struct { type NewInstallOverlayFunc func(installDir, provider string) (InstallOverlay, error) +type KeyBrokerService struct { + cloudProvider string // Cloud provider +} + var NewInstallOverlayFunctions = make(map[string]NewInstallOverlayFunc) // InstallOverlay defines common operations to an install overlay (install/overlays/*) @@ -64,6 +71,42 @@ type InstallOverlay interface { Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error } +func NewKeyBrokerService(provider string) (*KeyBrokerService, error) { + // Clone kbs repo + repoURL := "https://github.com/confidential-containers/kbs" + cmd := exec.Command("git", "clone", repoURL) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + fmt.Printf("Error running git clone: %v\n", err) + return nil, err + } + + // Create secret + content := []byte("This is my super secret") + filePath := "kbs/config/kubernetes/overlays/key.bin" + // Create the file. + file, err := os.Create(filePath) + if err != nil { + fmt.Printf("Error creating file: %v\n", err) + return nil, err + } + defer file.Close() + + // Write the content to the file. + _, err = file.Write(content) + if err != nil { + fmt.Printf("Error writing to file: %v\n", err) + return nil, err + } + + return &KeyBrokerService{ + cloudProvider: provider, + }, nil +} + func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, error) { namespace := "confidential-containers-system" @@ -121,6 +164,143 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro return overlayFunc(installDir, provider) } +// TODO: Use kustomize overlay to update this file +func UpdateKbsKustomizationFile(imagePath string, imageTag string) error { + // Read the content of the existing kustomization.yaml file. + filePath := "base/kustomization.yaml" + content, err := ioutil.ReadFile(filePath) + if err != nil { + fmt.Printf("Error reading kustomization file: %v\n", err) + return err + } + + // Convert the content to a string. + kustomizationContent := string(content) + + // Define the values to update. + kustomizationContent = strings.Replace(kustomizationContent, "newName: ghcr.io/confidential-containers/key-broker-service", "newName: "+imagePath, -1) + kustomizationContent = strings.Replace(kustomizationContent, "newTag: built-in-as-v0.7.0", "newTag: "+imageTag, -1) + + // Write the updated content back to the same file. + err = ioutil.WriteFile(filePath, []byte(kustomizationContent), 0644) + if err != nil { + fmt.Printf("Error writing to kustomization file: %v\n", err) + return err + } + + fmt.Println("Kustomization file updated successfully.") + return nil + +} + +func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTag string) error { + originalDir, err := os.Getwd() + if err != nil { + fmt.Printf("Error getting the current working directory: %v\n", err) + return err + } + + // jump to kbs kubernetes config directory + newDirectory := "kbs/config/kubernetes/" + err = os.Chdir(newDirectory) + if err != nil { + fmt.Printf("Error changing the working directory: %v\n", err) + return err + } + + // Note: Use kustomize overlay to update this + err = UpdateKbsKustomizationFile(imagePath, imageTag) + if err != nil { + fmt.Printf("Error updating kustomization file: %v\n", err) + return err + } + + // Deploy kbs + k8sCnfDir, err := os.Getwd() + if err != nil { + fmt.Printf("Error getting the current working directory: %v\n", err) + return err + } + fmt.Println(k8sCnfDir) + + keyFile := filepath.Join(k8sCnfDir, "overlays/key.bin") + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + fmt.Println("key.bin file does not exist") + //return err + } + + kbsCert := filepath.Join(k8sCnfDir, "base/kbs.pem") + if _, err := os.Stat(kbsCert); os.IsNotExist(err) { + kbsKey := filepath.Join(k8sCnfDir, "base/kbs.key") + keyOutputFile, err := os.Create(kbsKey) + if err != nil { + fmt.Printf("Error creating key file: %v\n", err) + os.Exit(1) + } + defer keyOutputFile.Close() + + opensslGenPKeyCmd := exec.Command("openssl", "genpkey", "-algorithm", "ed25519") + opensslGenPKeyCmd.Stdout = keyOutputFile + opensslGenPKeyCmd.Stderr = os.Stderr + fmt.Printf("Running command: %v\n", opensslGenPKeyCmd.Args) + if err := opensslGenPKeyCmd.Run(); err != nil { + fmt.Printf("Error generating key: %v\n", err) + return err + } + + opensslPKeyCmd := exec.Command("openssl", "pkey", "-in", kbsKey, "-pubout", "-out", kbsCert) + opensslPKeyCmd.Stdout = os.Stdout + opensslPKeyCmd.Stderr = os.Stderr + if err := opensslPKeyCmd.Run(); err != nil { + fmt.Printf("Error creating kbs.pem: %v\n", err) + return err + } + } + + kubectlApplyCmd := exec.Command("kubectl", "apply", "-k", k8sCnfDir+"/overlays") + kubectlApplyCmd.Stdout = os.Stdout + kubectlApplyCmd.Stderr = os.Stderr + if err := kubectlApplyCmd.Run(); err != nil { + fmt.Printf("Error running 'kubectl apply': %v\n", err) + return err + } + + // Return to the original working directory. + err = os.Chdir(originalDir) + if err != nil { + fmt.Printf("Error changing back to the original working directory: %v\n", err) + return err + } + + // remove kbs repo + directoryPath := "kbs" + + err = os.RemoveAll(directoryPath) + if err != nil { + fmt.Printf("Error deleting directory: %v\n", err) + return err + } + + return nil +} + +func (p *KeyBrokerService) Delete(ctx context.Context) error { + // Remove kbs deployment + k8sCnfDir := "kbs/config/kubernetes" + kubectlDeleteCmd := exec.Command("kubectl", "delete", "-k", k8sCnfDir+"/overlays") + kubectlDeleteCmd.Stdout = os.Stdout + kubectlDeleteCmd.Stderr = os.Stderr + + err := kubectlDeleteCmd.Run() + if err != nil { + fmt.Printf("Error running 'kubectl delete': %v\n", err) + return err + } + + return nil +} + + // Deletes the peer pods installation including the controller manager. func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error { client, err := cfg.NewClient() From c222428995f994904ef73bf693743c524035d465 Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Tue, 17 Oct 2023 17:33:26 +0530 Subject: [PATCH 2/3] Provisioner: Add support to fetch kbs service ip Fixes: #1471 Signed-off-by: Kartik Joshi --- install/overlays/azure/kustomization.yaml | 1 + test/e2e/main_test.go | 43 ++- .../azure/provision_azure.properties | 2 + test/provisioner/azure/provision_common.go | 4 +- .../azure/provision_initializer.go | 4 + test/provisioner/provision.go | 334 ++++++++++++------ 6 files changed, 270 insertions(+), 118 deletions(-) diff --git a/install/overlays/azure/kustomization.yaml b/install/overlays/azure/kustomization.yaml index 3bf3b1666..503c9ea31 100644 --- a/install/overlays/azure/kustomization.yaml +++ b/install/overlays/azure/kustomization.yaml @@ -32,6 +32,7 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name + - AA_KBC_PARAMS="" #set #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index a3eb3c8cc..5dc326c22 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -5,6 +5,7 @@ package e2e import ( "context" + "fmt" "os" "testing" @@ -17,12 +18,9 @@ import ( ) var ( - testEnv env.Environment - cloudProvider string - provisioner pv.CloudProvisioner - testEnv env.Environment - cloudProvider string - provisioner pv.CloudProvisioner + testEnv env.Environment + cloudProvider string + provisioner pv.CloudProvisioner keyBrokerService *pv.KeyBrokerService ) @@ -84,9 +82,6 @@ func TestMain(m *testing.M) { // the VPC images storage. podvmImage := os.Getenv("TEST_PODVM_IMAGE") - kbsImage := os.Getenv("TEST_KBS_IMAGE") - kbsImageTag := os.Getenv("TEST_KBS_IMAGE_TAG") - // The TEST_PROVISION_FILE is an optional variable which specifies the path // to the provision properties file. The file must have the format: // @@ -101,9 +96,9 @@ func TestMain(m *testing.M) { } // The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA - shouldDeployKbs := false - if os.Getenv("DEPLOY_KBS") == "yes" { - shouldDeployKbs = true + shouldDeployKbs := true + if os.Getenv("DEPLOY_KBS") != "yes" { + shouldDeployKbs = false } if !shouldProvisionCluster { @@ -121,6 +116,7 @@ func TestMain(m *testing.M) { testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { log.Info("Do setup") var err error + // Get properties if shouldProvisionCluster { log.Info("Cluster provisioning") @@ -133,16 +129,28 @@ func TestMain(m *testing.M) { } } + props := provisioner.GetProperties(ctx, cfg) + var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + return ctx, fmt.Errorf("kbs image not provided") + } - if keyBrokerService, err = pv.NewKeyBrokerService(cloudProvider); err != nil { + if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil { return ctx, err } - if err = keyBrokerService.Deploy(ctx, kbsImage, kbsImageTag); err != nil { + if err = keyBrokerService.Deploy(ctx, cfg, props); err != nil { + return ctx, err + } + var kbsPodIP string + if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { return ctx, err } + + kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080" + log.Infof("KBS PARAMS%s:", kbsparams) } if podvmImage != "" { @@ -158,8 +166,11 @@ func TestMain(m *testing.M) { if cloudAPIAdaptor, err = pv.NewCloudAPIAdaptor(cloudProvider, relativeInstallDirectory); err != nil { return ctx, err } + + props = provisioner.GetProperties(ctx, cfg) + props["AA_KBC_PARAMS"] = kbsparams log.Info("Deploy the Cloud API Adaptor") - if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil { + if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil { return ctx, err } } @@ -199,7 +210,7 @@ func TestMain(m *testing.M) { } if shouldDeployKbs { - if err = keyBrokerService.Delete(ctx); err != nil { + if err = keyBrokerService.Delete(ctx, cfg); err != nil { return ctx, err } } diff --git a/test/provisioner/azure/provision_azure.properties b/test/provisioner/azure/provision_azure.properties index 6f6c3e6db..0b19cff4c 100644 --- a/test/provisioner/azure/provision_azure.properties +++ b/test/provisioner/azure/provision_azure.properties @@ -13,3 +13,5 @@ SSH_USERNAME="" AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" +KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs" +KBS_IMAGE_TAG="latest" diff --git a/test/provisioner/azure/provision_common.go b/test/provisioner/azure/provision_common.go index 8483b218d..d6bdf3ede 100644 --- a/test/provisioner/azure/provision_common.go +++ b/test/provisioner/azure/provision_common.go @@ -360,6 +360,8 @@ func getPropertiesImpl() map[string]string { "AZURE_IMAGE_ID": AzureProps.ImageID, "AZURE_SUBNET_ID": AzureProps.SubnetID, "AZURE_INSTANCE_SIZE": AzureProps.InstanceSize, + "KBS_IMAGE": AzureProps.KbsImage, + "KBS_IMAGE_TAG": AzureProps.KbsImageTag, } return props @@ -378,7 +380,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex func isAzureKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME": + case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "AA_KBC_PARAMS": return true default: return false diff --git a/test/provisioner/azure/provision_initializer.go b/test/provisioner/azure/provision_initializer.go index 9a0c75184..fa1ab0c78 100644 --- a/test/provisioner/azure/provision_initializer.go +++ b/test/provisioner/azure/provision_initializer.go @@ -33,6 +33,8 @@ type AzureProperties struct { IsCIManaged bool CaaImage string IsSelfManaged bool + KbsImage string + KbsImageTag string InstanceSize string NodeName string @@ -64,6 +66,8 @@ func initAzureProperties(properties map[string]string) error { SshUserName: properties["SSH_USERNAME"], ManagedIdentityName: properties["MANAGED_IDENTITY_NAME"], CaaImage: properties["CAA_IMAGE"], + KbsImage: properties["KBS_IMAGE"], + KbsImageTag: properties["KBS_IMAGE_TAG"], } CIManagedStr := properties["IS_CI_MANAGED_CLUSTER"] diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index 4e220e688..5420b6fd9 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -5,14 +5,16 @@ package provisioner import ( "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" "fmt" + "io" "os" "os/exec" "path/filepath" "time" - "strings" - "path/filepath" - "io/ioutil" "github.com/BurntSushi/toml" log "github.com/sirupsen/logrus" @@ -40,6 +42,11 @@ type CloudProvisioner interface { type NewProvisionerFunc func(properties map[string]string) (CloudProvisioner, error) +// KbsInstallOverlay implements the InstallOverlay interface +type KbsInstallOverlay struct { + overlay *KustomizeOverlay +} + var NewProvisionerFunctions = make(map[string]NewProvisionerFunc) type CloudAPIAdaptor struct { @@ -56,7 +63,7 @@ type CloudAPIAdaptor struct { type NewInstallOverlayFunc func(installDir, provider string) (InstallOverlay, error) type KeyBrokerService struct { - cloudProvider string // Cloud provider + installOverlay InstallOverlay // Pointer to the kustomize overlay } var NewInstallOverlayFunctions = make(map[string]NewInstallOverlayFunc) @@ -71,39 +78,131 @@ type InstallOverlay interface { Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error } -func NewKeyBrokerService(provider string) (*KeyBrokerService, error) { +func runCommand(command string, stdout io.Writer, stderr io.Writer, args ...string) error { + cmd := exec.Command(command, args...) + cmd.Stdout = stdout + cmd.Stderr = stderr + + fmt.Printf("Running command: %s %v\n", command, args) + + if err := cmd.Run(); err != nil { + err = fmt.Errorf(fmt.Sprintf("Error running command: %s %v - %s", command, args, err)) + + log.Errorf("%v", err) + return err + } + + return nil +} + +func saveToFile(filename string, content []byte) error { + // Save contents to file + err := os.WriteFile(filename, content, 0644) + if err != nil { + return fmt.Errorf("error writing contents to file: %w", err) + } + return nil +} + +func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Clone kbs repo repoURL := "https://github.com/confidential-containers/kbs" - cmd := exec.Command("git", "clone", repoURL) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err := cmd.Run() - if err != nil { - fmt.Printf("Error running git clone: %v\n", err) + if err := runCommand("git", os.Stdout, os.Stderr, "clone", repoURL); err != nil { return nil, err } + log.Info("creating key.bin") + // Create secret - content := []byte("This is my super secret") - filePath := "kbs/config/kubernetes/overlays/key.bin" + content := []byte("This is my cluster name: " + clusterName) + filePath := "kbs/kbs/config/kubernetes/overlays/key.bin" // Create the file. file, err := os.Create(filePath) if err != nil { - fmt.Printf("Error creating file: %v\n", err) + err = fmt.Errorf("Error creating file: %w\n", err) + log.Errorf("%v", err) return nil, err } defer file.Close() // Write the content to the file. - _, err = file.Write(content) + err = saveToFile(filePath, content) + if err != nil { + err = fmt.Errorf("Error writing to the file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + k8sCnfDir, err := os.Getwd() + if err != nil { + err = fmt.Errorf("Error getting the current working directory: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + fmt.Println(k8sCnfDir) + + kbsCert := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.pem") + if _, err := os.Stat(kbsCert); os.IsNotExist(err) { + kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key") + keyOutputFile, err := os.Create(kbsKey) + if err != nil { + err = fmt.Errorf("Error creating key file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + defer keyOutputFile.Close() + + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: privateKey, + }) + + // Save private key to file + err = saveToFile(kbsKey, privateKeyPEM) + if err != nil { + err = fmt.Errorf("Error saving private key to file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + publicKey := privateKey.Public().(ed25519.PublicKey) + publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + publicKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Bytes: publicKeyX509, + }) + + // Save public key to file + err = saveToFile(kbsCert, publicKeyPEM) + if err != nil { + err = fmt.Errorf("Error saving public key to file: %w\n", err) + log.Errorf("%v", err) + return nil, err + } + + } + + overlay, err := NewKbsInstallOverlay("kbs") if err != nil { - fmt.Printf("Error writing to file: %v\n", err) return nil, err } return &KeyBrokerService{ - cloudProvider: provider, + installOverlay: overlay, }, nil } @@ -129,7 +228,6 @@ func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, e // GetCloudProvisioner returns a CloudProvisioner implementation func GetCloudProvisioner(provider string, propertiesFile string) (CloudProvisioner, error) { - properties := make(map[string]string) if propertiesFile != "" { f, err := os.ReadFile(propertiesFile) @@ -164,111 +262,161 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro return overlayFunc(installDir, provider) } -// TODO: Use kustomize overlay to update this file -func UpdateKbsKustomizationFile(imagePath string, imageTag string) error { - // Read the content of the existing kustomization.yaml file. - filePath := "base/kustomization.yaml" - content, err := ioutil.ReadFile(filePath) +func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { + log.Info("Creating kbs install overlay") + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base")) if err != nil { - fmt.Printf("Error reading kustomization file: %v\n", err) - return err + return nil, err } - // Convert the content to a string. - kustomizationContent := string(content) + return &KbsInstallOverlay{ + overlay: overlay, + }, nil +} + +func (lio *KbsInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { + return lio.overlay.Apply(ctx, cfg) +} - // Define the values to update. - kustomizationContent = strings.Replace(kustomizationContent, "newName: ghcr.io/confidential-containers/key-broker-service", "newName: "+imagePath, -1) - kustomizationContent = strings.Replace(kustomizationContent, "newTag: built-in-as-v0.7.0", "newTag: "+imageTag, -1) +func (lio *KbsInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) error { + return lio.overlay.Delete(ctx, cfg) +} - // Write the updated content back to the same file. - err = ioutil.WriteFile(filePath, []byte(kustomizationContent), 0644) - if err != nil { - fmt.Printf("Error writing to kustomization file: %v\n", err) +func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, props map[string]string) error { + var err error + log.Infof("Updating kbs image with %q", props["KBS_IMAGE"]) + if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newName", props["KBS_IMAGE"]); err != nil { + return err + } + + log.Infof("Updating CAA image tag with %q", props["KBS_IMAGE_TAG"]) + if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { return err } - fmt.Println("Kustomization file updated successfully.") return nil +} +func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) (string, error) { + client, err := cfg.NewClient() + if err != nil { + return "", err + } + + namespace := "coco-tenant" + deploymentName := "kbs" + + err = AllPodsRunning(ctx, cfg, namespace) + if err != nil { + err = fmt.Errorf("All pods are not running: %w\n", err) + log.Errorf("%v", err) + return "", err + } + + resources := client.Resources(namespace) + + // Get the service associated with the deployment + serviceList := &corev1.ServiceList{} + err = resources.List(context.TODO(), serviceList) + if err != nil { + err = fmt.Errorf("Error listing services: %w\n", err) + log.Errorf("%v", err) + return "", err + } + + var matchingService *corev1.Service + for i := range serviceList.Items { + service := &serviceList.Items[i] + if service.Name == deploymentName { + matchingService = service + break + } + } + + if matchingService == nil { + return "", fmt.Errorf("No service with label selector found") + } + + fmt.Printf("KBS Service IP: %s\n", matchingService.Spec.ClusterIP) + return matchingService.Spec.ClusterIP, nil } -func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTag string) error { +func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { + log.Info("Customize the overlay yaml file") + if err := p.installOverlay.Edit(ctx, cfg, props); err != nil { + return err + } + originalDir, err := os.Getwd() if err != nil { - fmt.Printf("Error getting the current working directory: %v\n", err) + err = fmt.Errorf("Error getting the current working directory: %w\n", err) + log.Errorf("%v", err) return err } - // jump to kbs kubernetes config directory - newDirectory := "kbs/config/kubernetes/" + newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { - fmt.Printf("Error changing the working directory: %v\n", err) + err = fmt.Errorf("Error changing the working directory: %w\n", err) + log.Errorf("%v", err) return err } - // Note: Use kustomize overlay to update this - err = UpdateKbsKustomizationFile(imagePath, imageTag) + // Replace this to use install overlay + cmd := exec.Command("kubectl", "apply", "-k", "overlays") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) if err != nil { - fmt.Printf("Error updating kustomization file: %v\n", err) return err } - // Deploy kbs - k8sCnfDir, err := os.Getwd() + /* + log.Info("Install Kbs") + if err := p.installOverlay.Apply(ctx, cfg); err != nil { + return err + } + */ + + // Return to the original working directory. + err = os.Chdir(originalDir) if err != nil { - fmt.Printf("Error getting the current working directory: %v\n", err) + err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) + log.Errorf("%v", err) return err } - fmt.Println(k8sCnfDir) - keyFile := filepath.Join(k8sCnfDir, "overlays/key.bin") - if _, err := os.Stat(keyFile); os.IsNotExist(err) { - fmt.Println("key.bin file does not exist") - //return err - } + return nil +} - kbsCert := filepath.Join(k8sCnfDir, "base/kbs.pem") - if _, err := os.Stat(kbsCert); os.IsNotExist(err) { - kbsKey := filepath.Join(k8sCnfDir, "base/kbs.key") - keyOutputFile, err := os.Create(kbsKey) - if err != nil { - fmt.Printf("Error creating key file: %v\n", err) - os.Exit(1) - } - defer keyOutputFile.Close() +func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error { - opensslGenPKeyCmd := exec.Command("openssl", "genpkey", "-algorithm", "ed25519") - opensslGenPKeyCmd.Stdout = keyOutputFile - opensslGenPKeyCmd.Stderr = os.Stderr - fmt.Printf("Running command: %v\n", opensslGenPKeyCmd.Args) - if err := opensslGenPKeyCmd.Run(); err != nil { - fmt.Printf("Error generating key: %v\n", err) - return err - } + originalDir, err := os.Getwd() + if err != nil { + err = fmt.Errorf("Error getting the current working directory: %w\n", err) + log.Errorf("%v", err) + return err + } - opensslPKeyCmd := exec.Command("openssl", "pkey", "-in", kbsKey, "-pubout", "-out", kbsCert) - opensslPKeyCmd.Stdout = os.Stdout - opensslPKeyCmd.Stderr = os.Stderr - if err := opensslPKeyCmd.Run(); err != nil { - fmt.Printf("Error creating kbs.pem: %v\n", err) - return err - } + // Remove kbs deployment + newDirectory := "kbs/kbs/config/kubernetes" + err = os.Chdir(newDirectory) + if err != nil { + err = fmt.Errorf("Error changing the working directory: %w\n", err) + log.Errorf("%v", err) + return err } - kubectlApplyCmd := exec.Command("kubectl", "apply", "-k", k8sCnfDir+"/overlays") - kubectlApplyCmd.Stdout = os.Stdout - kubectlApplyCmd.Stderr = os.Stderr - if err := kubectlApplyCmd.Run(); err != nil { - fmt.Printf("Error running 'kubectl apply': %v\n", err) + log.Info("Delete Kbs deployment") + if err := p.installOverlay.Delete(ctx, cfg); err != nil { return err } // Return to the original working directory. err = os.Chdir(originalDir) if err != nil { - fmt.Printf("Error changing back to the original working directory: %v\n", err) + err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) + log.Errorf("%v", err) return err } @@ -277,30 +425,14 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTa err = os.RemoveAll(directoryPath) if err != nil { - fmt.Printf("Error deleting directory: %v\n", err) - return err - } - - return nil -} - -func (p *KeyBrokerService) Delete(ctx context.Context) error { - // Remove kbs deployment - k8sCnfDir := "kbs/config/kubernetes" - kubectlDeleteCmd := exec.Command("kubectl", "delete", "-k", k8sCnfDir+"/overlays") - kubectlDeleteCmd.Stdout = os.Stdout - kubectlDeleteCmd.Stderr = os.Stderr - - err := kubectlDeleteCmd.Run() - if err != nil { - fmt.Printf("Error running 'kubectl delete': %v\n", err) + err = fmt.Errorf("Error deleting directory: %w\n", err) + log.Errorf("%v", err) return err } return nil } - // Deletes the peer pods installation including the controller manager. func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error { client, err := cfg.NewClient() @@ -502,7 +634,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string) for _, o := range metaList { obj, _ := o.(k8s.Object) fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName()) - if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*6)); err != nil { + if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil { return err } fmt.Printf("pod '%s' is Ready\n", obj.GetName()) From 06ceae0e5819875a4d0e7fb9ec94c8090926fdee Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Thu, 25 Jan 2024 15:29:55 +0530 Subject: [PATCH 3/3] Provisioner: Add kbs provisioner in provisioner-cli Fixes: #1471 Signed-off-by: Kartik Joshi --- install/overlays/azure/kustomization.yaml | 3 +- test/e2e/main_test.go | 14 +- .../azure/provision_azure.properties | 4 +- test/provisioner/provision.go | 166 +++++------------- test/tools/provisioner-cli/main.go | 41 ++++- 5 files changed, 97 insertions(+), 131 deletions(-) diff --git a/install/overlays/azure/kustomization.yaml b/install/overlays/azure/kustomization.yaml index 503c9ea31..f9a6a0947 100644 --- a/install/overlays/azure/kustomization.yaml +++ b/install/overlays/azure/kustomization.yaml @@ -32,13 +32,12 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name - - AA_KBC_PARAMS="" #set + - AA_KBC_PARAMS="" #set KBC params for podvm #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 #- AZURE_INSTANCE_SIZES="" # comma separated #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm - #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150 ##TLS_SETTINGS #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 5dc326c22..d2d77ef88 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -116,7 +116,12 @@ func TestMain(m *testing.M) { testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { log.Info("Do setup") var err error + // Get properties + props := provisioner.GetProperties(ctx, cfg) + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + return ctx, fmt.Errorf("kbs image not provided") + } if shouldProvisionCluster { log.Info("Cluster provisioning") @@ -129,14 +134,9 @@ func TestMain(m *testing.M) { } } - props := provisioner.GetProperties(ctx, cfg) var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") - if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { - return ctx, fmt.Errorf("kbs image not provided") - } - if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil { return ctx, err } @@ -145,11 +145,11 @@ func TestMain(m *testing.M) { return ctx, err } var kbsPodIP string - if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { + if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil { return ctx, err } - kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080" + kbsparams = "cc_kbc::http://" + kbsPodIP + ":8080" log.Infof("KBS PARAMS%s:", kbsparams) } diff --git a/test/provisioner/azure/provision_azure.properties b/test/provisioner/azure/provision_azure.properties index 0b19cff4c..5e8bf51e0 100644 --- a/test/provisioner/azure/provision_azure.properties +++ b/test/provisioner/azure/provision_azure.properties @@ -13,5 +13,5 @@ SSH_USERNAME="" AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" -KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs" -KBS_IMAGE_TAG="latest" +KBS_IMAGE="${KBS_IMAGE}" +KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index 5420b6fd9..71d2ca7a8 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -10,7 +10,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -78,40 +77,19 @@ type InstallOverlay interface { Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error } -func runCommand(command string, stdout io.Writer, stderr io.Writer, args ...string) error { - cmd := exec.Command(command, args...) - cmd.Stdout = stdout - cmd.Stderr = stderr - - fmt.Printf("Running command: %s %v\n", command, args) - - if err := cmd.Run(); err != nil { - err = fmt.Errorf(fmt.Sprintf("Error running command: %s %v - %s", command, args, err)) - - log.Errorf("%v", err) - return err - } - - return nil -} +// Waiting timeout for bringing up the pod +const PodWaitTimeout = time.Second * 30 func saveToFile(filename string, content []byte) error { // Save contents to file err := os.WriteFile(filename, content, 0644) if err != nil { - return fmt.Errorf("error writing contents to file: %w", err) + return fmt.Errorf("writing contents to file: %w", err) } return nil } func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { - // Clone kbs repo - repoURL := "https://github.com/confidential-containers/kbs" - - if err := runCommand("git", os.Stdout, os.Stderr, "clone", repoURL); err != nil { - return nil, err - } - log.Info("creating key.bin") // Create secret @@ -120,7 +98,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Create the file. file, err := os.Create(filePath) if err != nil { - err = fmt.Errorf("Error creating file: %w\n", err) + err = fmt.Errorf("creating file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -129,14 +107,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Write the content to the file. err = saveToFile(filePath, content) if err != nil { - err = fmt.Errorf("Error writing to the file: %w\n", err) + err = fmt.Errorf("writing to the file: %w\n", err) log.Errorf("%v", err) return nil, err } k8sCnfDir, err := os.Getwd() if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) + err = fmt.Errorf("getting the current working directory: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -147,7 +125,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key") keyOutputFile, err := os.Create(kbsKey) if err != nil { - err = fmt.Errorf("Error creating key file: %w\n", err) + err = fmt.Errorf("creating key file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -155,7 +133,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { _, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { - err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err) + err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -168,7 +146,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Save private key to file err = saveToFile(kbsKey, privateKeyPEM) if err != nil { - err = fmt.Errorf("Error saving private key to file: %w\n", err) + err = fmt.Errorf("saving private key to file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -176,7 +154,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { publicKey := privateKey.Public().(ed25519.PublicKey) publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) if err != nil { - err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err) + err = fmt.Errorf("generating Ed25519 public key: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -189,14 +167,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Save public key to file err = saveToFile(kbsCert, publicKeyPEM) if err != nil { - err = fmt.Errorf("Error saving public key to file: %w\n", err) + err = fmt.Errorf("saving public key to file: %w\n", err) log.Errorf("%v", err) return nil, err } } - overlay, err := NewKbsInstallOverlay("kbs") + overlay, err := NewBaseKbsInstallOverlay("kbs") if err != nil { return nil, err } @@ -262,7 +240,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro return overlayFunc(installDir, provider) } -func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { +func NewBaseKbsInstallOverlay(installDir string) (InstallOverlay, error) { log.Info("Creating kbs install overlay") overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base")) if err != nil { @@ -274,6 +252,18 @@ func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { }, nil } +func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { + log.Info("Creating kbs install overlay") + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/overlays")) + if err != nil { + return nil, err + } + + return &KbsInstallOverlay{ + overlay: overlay, + }, nil +} + func (lio *KbsInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { return lio.overlay.Apply(ctx, cfg) } @@ -289,7 +279,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro return err } - log.Infof("Updating CAA image tag with %q", props["KBS_IMAGE_TAG"]) + log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"]) if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { return err } @@ -297,7 +287,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro return nil } -func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) (string, error) { +func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) { client, err := cfg.NewClient() if err != nil { return "", err @@ -314,31 +304,29 @@ func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) } resources := client.Resources(namespace) - - // Get the service associated with the deployment - serviceList := &corev1.ServiceList{} - err = resources.List(context.TODO(), serviceList) + podList := &corev1.PodList{} + err = resources.List(context.TODO(), podList) if err != nil { - err = fmt.Errorf("Error listing services: %w\n", err) + err = fmt.Errorf("Error listing pods: %w\n", err) log.Errorf("%v", err) return "", err } - var matchingService *corev1.Service - for i := range serviceList.Items { - service := &serviceList.Items[i] - if service.Name == deploymentName { - matchingService = service + var matchingPod *corev1.Pod + for i := range podList.Items { + pod := &podList.Items[i] + if pod.Labels["app"] == deploymentName { + matchingPod = pod break } } - if matchingService == nil { - return "", fmt.Errorf("No service with label selector found") + if matchingPod == nil { + return "", fmt.Errorf("No pod with label selector found") } - fmt.Printf("KBS Service IP: %s\n", matchingService.Spec.ClusterIP) - return matchingService.Spec.ClusterIP, nil + fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP) + return matchingPod.Status.PodIP, nil } func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error { @@ -347,42 +335,14 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } - originalDir, err := os.Getwd() - if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - newDirectory := "kbs/kbs/config/kubernetes" - err = os.Chdir(newDirectory) - if err != nil { - err = fmt.Errorf("Error changing the working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - // Replace this to use install overlay - cmd := exec.Command("kubectl", "apply", "-k", "overlays") - cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) - stdoutStderr, err := cmd.CombinedOutput() - log.Tracef("%v, output: %s", cmd, stdoutStderr) + // Create kustomize pointer for overlay directory with updated changes + tmpoverlay, err := NewKbsInstallOverlay("kbs") if err != nil { return err } - /* - log.Info("Install Kbs") - if err := p.installOverlay.Apply(ctx, cfg); err != nil { - return err - } - */ - - // Return to the original working directory. - err = os.Chdir(originalDir) - if err != nil { - err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) - log.Errorf("%v", err) + log.Info("Install Kbs") + if err := tmpoverlay.Apply(ctx, cfg); err != nil { return err } @@ -390,46 +350,16 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop } func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error { - - originalDir, err := os.Getwd() - if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - // Remove kbs deployment - newDirectory := "kbs/kbs/config/kubernetes" - err = os.Chdir(newDirectory) - if err != nil { - err = fmt.Errorf("Error changing the working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - log.Info("Delete Kbs deployment") - if err := p.installOverlay.Delete(ctx, cfg); err != nil { - return err - } - - // Return to the original working directory. - err = os.Chdir(originalDir) + // Create kustomize pointer for overlay directory with updated changes + tmpoverlay, err := NewKbsInstallOverlay("kbs") if err != nil { - err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) - log.Errorf("%v", err) return err } - // remove kbs repo - directoryPath := "kbs" - - err = os.RemoveAll(directoryPath) - if err != nil { - err = fmt.Errorf("Error deleting directory: %w\n", err) - log.Errorf("%v", err) + log.Info("Uninstall the cloud-api-adaptor") + if err = tmpoverlay.Delete(ctx, cfg); err != nil { return err } - return nil } @@ -634,7 +564,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string) for _, o := range metaList { obj, _ := o.(k8s.Object) fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName()) - if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil { + if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(PodWaitTimeout)); err != nil { return err } fmt.Printf("pod '%s' is Ready\n", obj.GetName()) diff --git a/test/tools/provisioner-cli/main.go b/test/tools/provisioner-cli/main.go index 3e3e85421..277e3683c 100644 --- a/test/tools/provisioner-cli/main.go +++ b/test/tools/provisioner-cli/main.go @@ -63,6 +63,11 @@ func main() { } if *action == "provision" { + props := provisioner.GetProperties(context.TODO(), cfg) + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + log.Fatal("kbs image not provided") + } + log.Info("Creating VPC...") if err := provisioner.CreateVPC(context.TODO(), cfg); err != nil { log.Fatal(err) @@ -83,11 +88,33 @@ func main() { } } + log.Info("Deploying kbs") + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + if err != nil { + log.Fatal(err) + } + + if err = keyBrokerService.Deploy(context.TODO(), cfg, props); err != nil { + log.Fatal(err) + } + + var kbsPodIP string + kbsPodIP, err = keyBrokerService.GetKbsPodIP(context.TODO(), cfg) + if err != nil { + log.Fatal(err) + } + + kbsparams := "cc_kbc::http://" + kbsPodIP + ":8080" + log.Infof("KBS PARAMS: %s", kbsparams) + + props = provisioner.GetProperties(context.TODO(), cfg) + props["AA_KBC_PARAMS"] = kbsparams + cloudAPIAdaptor, err := pv.NewCloudAPIAdaptor(cloudProvider, installDirectory) if err != nil { log.Fatal(err) } - if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)); err != nil { + if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, props); err != nil { log.Fatal(err) } } @@ -128,7 +155,7 @@ func main() { if kubeconfigPath == "" { log.Fatal("Unabled to find a kubeconfig file") } - cfg := envconf.NewWithKubeConfig(kubeconfigPath) + cfg = envconf.NewWithKubeConfig(kubeconfigPath) err = deployer.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)) if err != nil { @@ -143,6 +170,16 @@ func main() { log.Fatal(err) } + props := provisioner.GetProperties(context.TODO(), cfg) + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + if err != nil { + log.Fatal(err) + } + + if err = keyBrokerService.Delete(context.TODO(), cfg); err != nil { + log.Fatal(err) + } + kubeconfigPath := kconf.ResolveKubeConfigFile() if kubeconfigPath == "" { log.Fatal("Unabled to find a kubeconfig file")