Provisioner: Add kbs provisioner in provisioner-cli

Fixes: #1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>

install/overlays/azure/kustomization.yaml

@@ -32,13 +32,12 @@ configMapGenerator:
   # /subscriptions/<AZURE_SUBSCRIPTION_ID>/resourceGroups/<AZURE_RESOURCE_GROUP>/providers/Microsoft.Compute/images/<AZURE_IMAGE>
   - AZURE_IMAGE_ID="" #set
   - SSH_USERNAME="" #set peer pod vm admin user name
-  - AA_KBC_PARAMS="" #set
+  - AA_KBC_PARAMS="" #set KBC params for podvm
   #- DISABLECVM="" # Uncomment it if you want a generic VM
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
   #- AZURE_INSTANCE_SIZES="" # comma separated
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
-  #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS

test/e2e/main_test.go

@@ -118,6 +118,11 @@ func TestMain(m *testing.M) {
 		var err error
 		// Get properties
 
+		props := provisioner.GetProperties(ctx, cfg)
+		if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
+			return ctx, fmt.Errorf("kbs image not provided")
+		}
+
 		if shouldProvisionCluster {
 			log.Info("Cluster provisioning")
 			if err = provisioner.CreateVPC(ctx, cfg); err != nil {
@@ -129,27 +134,22 @@ func TestMain(m *testing.M) {
 			}
 		}
 
-		props := provisioner.GetProperties(ctx, cfg)
 		var kbsparams string
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
-			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
-				return ctx, fmt.Errorf("kbs image not provided")
-			}
-
 			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil {
 				return ctx, err
 			}
 
 			if err = keyBrokerService.Deploy(ctx, cfg, props); err != nil {
 				return ctx, err
 			}
-			var kbsPodIP string
-			if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
+			var kbsSvcIP string
+			if kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
 				return ctx, err
 			}
 
-			kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080"
+			kbsparams = "cc_kbc::http:" + kbsSvcIP + ":8080"
 			log.Infof("KBS PARAMS%s:", kbsparams)
 		}
 

test/provisioner/azure/provision_azure.properties

@@ -13,5 +13,5 @@ SSH_USERNAME=""
 AZURE_CLI_AUTH="false"
 IS_CI_MANAGED_CLUSTER="false"
 IS_SELF_MANAGED_CLUSTER="false"
-KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs"
-KBS_IMAGE_TAG="latest"
+KBS_IMAGE="${KBS_IMAGE}"
+KBS_IMAGE_TAG="${KBS_IMAGE_TAG}"

test/provisioner/provision.go

@@ -81,7 +81,7 @@ func saveToFile(filename string, content []byte) error {
 	// Save contents to file
 	err := os.WriteFile(filename, content, 0644)
 	if err != nil {
-		return fmt.Errorf("error writing contents to file: %w", err)
+		return fmt.Errorf("writing contents to file: %w", err)
 	}
 	return nil
 }
@@ -95,7 +95,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	// Create the file.
 	file, err := os.Create(filePath)
 	if err != nil {
-		err = fmt.Errorf("Error creating file: %w\n", err)
+		err = fmt.Errorf("creating file: %w\n", err)
 		log.Errorf("%v", err)
 		return nil, err
 	}
@@ -104,14 +104,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	// Write the content to the file.
 	err = saveToFile(filePath, content)
 	if err != nil {
-		err = fmt.Errorf("Error writing to the file: %w\n", err)
+		err = fmt.Errorf("writing to the file: %w\n", err)
 		log.Errorf("%v", err)
 		return nil, err
 	}
 
 	k8sCnfDir, err := os.Getwd()
 	if err != nil {
-		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
+		err = fmt.Errorf("getting the current working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return nil, err
 	}
@@ -122,15 +122,15 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key")
 		keyOutputFile, err := os.Create(kbsKey)
 		if err != nil {
-			err = fmt.Errorf("Error creating key file: %w\n", err)
+			err = fmt.Errorf("creating key file: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 		defer keyOutputFile.Close()
 
 		_, privateKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
-			err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err)
+			err = fmt.Errorf("generating Ed25519 key pair: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
@@ -143,15 +143,15 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		// Save private key to file
 		err = saveToFile(kbsKey, privateKeyPEM)
 		if err != nil {
-			err = fmt.Errorf("Error saving private key to file: %w\n", err)
+			err = fmt.Errorf("saving private key to file: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
 		publicKey := privateKey.Public().(ed25519.PublicKey)
 		publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey)
 		if err != nil {
-			err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err)
+			err = fmt.Errorf("generating Ed25519 public key: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
@@ -164,7 +164,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		// Save public key to file
 		err = saveToFile(kbsCert, publicKeyPEM)
 		if err != nil {
-			err = fmt.Errorf("Error saving public key to file: %w\n", err)
+			err = fmt.Errorf("saving public key to file: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
@@ -264,7 +264,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro
 		return err
 	}
 
-	log.Infof("Updating CAA image tag with %q", props["KBS_IMAGE_TAG"])
+	log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"])
 	if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil {
 		return err
 	}
@@ -294,7 +294,7 @@ func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config)
 	serviceList := &corev1.ServiceList{}
 	err = resources.List(context.TODO(), serviceList)
 	if err != nil {
-		err = fmt.Errorf("Error listing services: %w\n", err)
+		err = fmt.Errorf("listing services: %w\n", err)
 		log.Errorf("%v", err)
 		return "", err
 	}
@@ -324,20 +324,19 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
 
 	originalDir, err := os.Getwd()
 	if err != nil {
-		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
+		err = fmt.Errorf("getting the current working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return err
 	}
 
 	newDirectory := "kbs/kbs/config/kubernetes"
 	err = os.Chdir(newDirectory)
 	if err != nil {
-		err = fmt.Errorf("Error changing the working directory: %w\n", err)
+		err = fmt.Errorf("changing the working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return err
 	}
 
-	// Replace this to use install overlay
 	cmd := exec.Command("kubectl", "apply", "-k", "overlays")
 	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
 	stdoutStderr, err := cmd.CombinedOutput()
@@ -346,17 +345,10 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
 		return err
 	}
 
-	/*
-		log.Info("Install Kbs")
-		if err := p.installOverlay.Apply(ctx, cfg); err != nil {
-			return err
-		}
-	*/
-
 	// Return to the original working directory.
 	err = os.Chdir(originalDir)
 	if err != nil {
-		err = fmt.Errorf("Error changing back to the original working directory: %w\n", err)
+		err = fmt.Errorf("changing back to the original working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return err
 	}
@@ -368,7 +360,7 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro
 
 	originalDir, err := os.Getwd()
 	if err != nil {
-		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
+		err = fmt.Errorf("getting the current working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return err
 	}
@@ -377,20 +369,25 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro
 	newDirectory := "kbs/kbs/config/kubernetes"
 	err = os.Chdir(newDirectory)
 	if err != nil {
-		err = fmt.Errorf("Error changing the working directory: %w\n", err)
+		err = fmt.Errorf("changing the working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return err
 	}
 
+
 	log.Info("Delete Kbs deployment")
-	if err := p.installOverlay.Delete(ctx, cfg); err != nil {
+	cmd := exec.Command("kubectl", "delete", "-k", "overlays")
+	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
+	stdoutStderr, err := cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
 		return err
 	}
 
 	// Return to the original working directory.
 	err = os.Chdir(originalDir)
 	if err != nil {
-		err = fmt.Errorf("Error changing back to the original working directory: %w\n", err)
+		err = fmt.Errorf("changing back to the original working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return err
 	}

test/tools/provisioner-cli/main.go

@@ -63,6 +63,11 @@ func main() {
 	}
 
 	if *action == "provision" {
+		props := provisioner.GetProperties(context.TODO(), cfg)
+		if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
+			log.Fatal("kbs image not provided")
+		}
+
 		log.Info("Creating VPC...")
 		if err := provisioner.CreateVPC(context.TODO(), cfg); err != nil {
 			log.Fatal(err)
@@ -83,11 +88,33 @@ func main() {
 			}
 		}
 
+		log.Info("Deploying kbs")
+		keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"])
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err = keyBrokerService.Deploy(context.TODO(), cfg, props); err != nil {
+			log.Fatal(err)
+		}
+
+		var kbsSvcIP string
+		kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(context.TODO(), cfg)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		kbsparams := "cc_kbc::http:" + kbsSvcIP + ":8080"
+		log.Infof("KBS PARAMS: %q:", kbsparams)
+
+		props = provisioner.GetProperties(context.TODO(), cfg)
+		props["AA_KBC_PARAMS"] = kbsparams
+
 		cloudAPIAdaptor, err := pv.NewCloudAPIAdaptor(cloudProvider, installDirectory)
 		if err != nil {
 			log.Fatal(err)
 		}
-		if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)); err != nil {
+		if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, props); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -128,7 +155,7 @@ func main() {
 		if kubeconfigPath == "" {
 			log.Fatal("Unabled to find a kubeconfig file")
 		}
-		cfg := envconf.NewWithKubeConfig(kubeconfigPath)
+		cfg = envconf.NewWithKubeConfig(kubeconfigPath)
 
 		err = deployer.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg))
 		if err != nil {
@@ -143,6 +170,16 @@ func main() {
 			log.Fatal(err)
 		}
 
+		props := provisioner.GetProperties(context.TODO(), cfg)
+		keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"])
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err = keyBrokerService.Delete(context.TODO(), cfg); err != nil {
+			log.Fatal(err)
+		}
+
 		kubeconfigPath := kconf.ResolveKubeConfigFile()
 		if kubeconfigPath == "" {
 			log.Fatal("Unabled to find a kubeconfig file")