From e5ec4ff5515b03cb128fc825bd84c94faf5e91a1 Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Wed, 31 Jan 2024 15:27:26 +0530 Subject: [PATCH] Provisioner: Add kbs provisioner in provisioner-cli Fixes: #1471 Signed-off-by: Kartik Joshi --- install/overlays/azure/kustomization.yaml | 3 +- test/e2e/main_test.go | 16 +++--- .../azure/provision_azure.properties | 4 +- test/provisioner/provision.go | 49 +++++++++---------- test/tools/provisioner-cli/main.go | 41 +++++++++++++++- 5 files changed, 73 insertions(+), 40 deletions(-) diff --git a/install/overlays/azure/kustomization.yaml b/install/overlays/azure/kustomization.yaml index 503c9ea31d..f9a6a09474 100644 --- a/install/overlays/azure/kustomization.yaml +++ b/install/overlays/azure/kustomization.yaml @@ -32,13 +32,12 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name - - AA_KBC_PARAMS="" #set + - AA_KBC_PARAMS="" #set KBC params for podvm #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 #- AZURE_INSTANCE_SIZES="" # comma separated #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm - #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150 ##TLS_SETTINGS #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 5dc326c228..a3ac20050e 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -118,6 +118,11 @@ func TestMain(m *testing.M) { var err error // Get properties + props := provisioner.GetProperties(ctx, cfg) + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + return ctx, fmt.Errorf("kbs image not provided") + } + if shouldProvisionCluster { log.Info("Cluster provisioning") if err = provisioner.CreateVPC(ctx, cfg); err != nil { @@ -129,14 +134,9 @@ func TestMain(m *testing.M) { } } - props := provisioner.GetProperties(ctx, cfg) var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") - if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { - return ctx, fmt.Errorf("kbs image not provided") - } - if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil { return ctx, err } @@ -144,12 +144,12 @@ func TestMain(m *testing.M) { if err = keyBrokerService.Deploy(ctx, cfg, props); err != nil { return ctx, err } - var kbsPodIP string - if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { + var kbsSvcIP string + if kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { return ctx, err } - kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080" + kbsparams = "cc_kbc::http:" + kbsSvcIP + ":8080" log.Infof("KBS PARAMS%s:", kbsparams) } diff --git a/test/provisioner/azure/provision_azure.properties b/test/provisioner/azure/provision_azure.properties index 0b19cff4c5..5e8bf51e01 100644 --- a/test/provisioner/azure/provision_azure.properties +++ b/test/provisioner/azure/provision_azure.properties @@ -13,5 +13,5 @@ SSH_USERNAME="" AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" -KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs" -KBS_IMAGE_TAG="latest" +KBS_IMAGE="${KBS_IMAGE}" +KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index 0f0d6602a9..e579967e01 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -81,7 +81,7 @@ func saveToFile(filename string, content []byte) error { // Save contents to file err := os.WriteFile(filename, content, 0644) if err != nil { - return fmt.Errorf("error writing contents to file: %w", err) + return fmt.Errorf("writing contents to file: %w", err) } return nil } @@ -95,7 +95,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Create the file. file, err := os.Create(filePath) if err != nil { - err = fmt.Errorf("Error creating file: %w\n", err) + err = fmt.Errorf("creating file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -104,14 +104,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Write the content to the file. err = saveToFile(filePath, content) if err != nil { - err = fmt.Errorf("Error writing to the file: %w\n", err) + err = fmt.Errorf("writing to the file: %w\n", err) log.Errorf("%v", err) return nil, err } k8sCnfDir, err := os.Getwd() if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) + err = fmt.Errorf("getting the current working directory: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -122,7 +122,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key") keyOutputFile, err := os.Create(kbsKey) if err != nil { - err = fmt.Errorf("Error creating key file: %w\n", err) + err = fmt.Errorf("creating key file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -130,7 +130,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { _, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { - err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err) + err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -143,7 +143,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Save private key to file err = saveToFile(kbsKey, privateKeyPEM) if err != nil { - err = fmt.Errorf("Error saving private key to file: %w\n", err) + err = fmt.Errorf("saving private key to file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -151,7 +151,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { publicKey := privateKey.Public().(ed25519.PublicKey) publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) if err != nil { - err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err) + err = fmt.Errorf("generating Ed25519 public key: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -164,7 +164,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Save public key to file err = saveToFile(kbsCert, publicKeyPEM) if err != nil { - err = fmt.Errorf("Error saving public key to file: %w\n", err) + err = fmt.Errorf("saving public key to file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -264,7 +264,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro return err } - log.Infof("Updating CAA image tag with %q", props["KBS_IMAGE_TAG"]) + log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"]) if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { return err } @@ -294,7 +294,7 @@ func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) serviceList := &corev1.ServiceList{} err = resources.List(context.TODO(), serviceList) if err != nil { - err = fmt.Errorf("Error listing services: %w\n", err) + err = fmt.Errorf("listing services: %w\n", err) log.Errorf("%v", err) return "", err } @@ -324,7 +324,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop originalDir, err := os.Getwd() if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) + err = fmt.Errorf("getting the current working directory: %w\n", err) log.Errorf("%v", err) return err } @@ -332,12 +332,11 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { - err = fmt.Errorf("Error changing the working directory: %w\n", err) + err = fmt.Errorf("changing the working directory: %w\n", err) log.Errorf("%v", err) return err } - // Replace this to use install overlay cmd := exec.Command("kubectl", "apply", "-k", "overlays") cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) stdoutStderr, err := cmd.CombinedOutput() @@ -346,17 +345,10 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } - /* - log.Info("Install Kbs") - if err := p.installOverlay.Apply(ctx, cfg); err != nil { - return err - } - */ - // Return to the original working directory. err = os.Chdir(originalDir) if err != nil { - err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) + err = fmt.Errorf("changing back to the original working directory: %w\n", err) log.Errorf("%v", err) return err } @@ -368,7 +360,7 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro originalDir, err := os.Getwd() if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) + err = fmt.Errorf("getting the current working directory: %w\n", err) log.Errorf("%v", err) return err } @@ -377,20 +369,25 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro newDirectory := "kbs/kbs/config/kubernetes" err = os.Chdir(newDirectory) if err != nil { - err = fmt.Errorf("Error changing the working directory: %w\n", err) + err = fmt.Errorf("changing the working directory: %w\n", err) log.Errorf("%v", err) return err } + log.Info("Delete Kbs deployment") - if err := p.installOverlay.Delete(ctx, cfg); err != nil { + cmd := exec.Command("kubectl", "delete", "-k", "overlays") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err := cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { return err } // Return to the original working directory. err = os.Chdir(originalDir) if err != nil { - err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) + err = fmt.Errorf("changing back to the original working directory: %w\n", err) log.Errorf("%v", err) return err } diff --git a/test/tools/provisioner-cli/main.go b/test/tools/provisioner-cli/main.go index 3e3e85421f..5aebbceb75 100644 --- a/test/tools/provisioner-cli/main.go +++ b/test/tools/provisioner-cli/main.go @@ -63,6 +63,11 @@ func main() { } if *action == "provision" { + props := provisioner.GetProperties(context.TODO(), cfg) + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + log.Fatal("kbs image not provided") + } + log.Info("Creating VPC...") if err := provisioner.CreateVPC(context.TODO(), cfg); err != nil { log.Fatal(err) @@ -83,11 +88,33 @@ func main() { } } + log.Info("Deploying kbs") + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + if err != nil { + log.Fatal(err) + } + + if err = keyBrokerService.Deploy(context.TODO(), cfg, props); err != nil { + log.Fatal(err) + } + + var kbsSvcIP string + kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(context.TODO(), cfg) + if err != nil { + log.Fatal(err) + } + + kbsparams := "cc_kbc::http:" + kbsSvcIP + ":8080" + log.Infof("KBS PARAMS: %q:", kbsparams) + + props = provisioner.GetProperties(context.TODO(), cfg) + props["AA_KBC_PARAMS"] = kbsparams + cloudAPIAdaptor, err := pv.NewCloudAPIAdaptor(cloudProvider, installDirectory) if err != nil { log.Fatal(err) } - if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)); err != nil { + if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, props); err != nil { log.Fatal(err) } } @@ -128,7 +155,7 @@ func main() { if kubeconfigPath == "" { log.Fatal("Unabled to find a kubeconfig file") } - cfg := envconf.NewWithKubeConfig(kubeconfigPath) + cfg = envconf.NewWithKubeConfig(kubeconfigPath) err = deployer.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)) if err != nil { @@ -143,6 +170,16 @@ func main() { log.Fatal(err) } + props := provisioner.GetProperties(context.TODO(), cfg) + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + if err != nil { + log.Fatal(err) + } + + if err = keyBrokerService.Delete(context.TODO(), cfg); err != nil { + log.Fatal(err) + } + kubeconfigPath := kconf.ResolveKubeConfigFile() if kubeconfigPath == "" { log.Fatal("Unabled to find a kubeconfig file")