Provisioner: Add kbs provisioner in provisioner-cli

Fixes: #1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>

install/overlays/azure/kustomization.yaml

@@ -32,13 +32,12 @@ configMapGenerator:
   # /subscriptions/<AZURE_SUBSCRIPTION_ID>/resourceGroups/<AZURE_RESOURCE_GROUP>/providers/Microsoft.Compute/images/<AZURE_IMAGE>
   - AZURE_IMAGE_ID="" #set
   - SSH_USERNAME="" #set peer pod vm admin user name
-  - AA_KBC_PARAMS="" #set
+  - AA_KBC_PARAMS="" #set KBC params for podvm
   #- DISABLECVM="" # Uncomment it if you want a generic VM
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
   #- AZURE_INSTANCE_SIZES="" # comma separated
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
-  #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS

test/e2e/main_test.go

@@ -116,7 +116,12 @@ func TestMain(m *testing.M) {
 	testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 		log.Info("Do setup")
 		var err error
+
 		// Get properties
+		props := provisioner.GetProperties(ctx, cfg)
+		if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
+			return ctx, fmt.Errorf("kbs image not provided")
+		}
 
 		if shouldProvisionCluster {
 			log.Info("Cluster provisioning")
@@ -129,27 +134,22 @@ func TestMain(m *testing.M) {
 			}
 		}
 
-		props := provisioner.GetProperties(ctx, cfg)
 		var kbsparams string
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
-			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
-				return ctx, fmt.Errorf("kbs image not provided")
-			}
-
 			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil {
 				return ctx, err
 			}
 
 			if err = keyBrokerService.Deploy(ctx, cfg, props); err != nil {
 				return ctx, err
 			}
-			var kbsPodIP string
-			if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
+			var kbsSvcIP string
+			if kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
 				return ctx, err
 			}
 
-			kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080"
+			kbsparams = "cc_kbc::http:" + kbsSvcIP + ":8080"
 			log.Infof("KBS PARAMS%s:", kbsparams)
 		}
 

test/provisioner/azure/provision_azure.properties

@@ -13,5 +13,5 @@ SSH_USERNAME=""
 AZURE_CLI_AUTH="false"
 IS_CI_MANAGED_CLUSTER="false"
 IS_SELF_MANAGED_CLUSTER="false"
-KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs"
-KBS_IMAGE_TAG="latest"
+KBS_IMAGE="${KBS_IMAGE}"
+KBS_IMAGE_TAG="${KBS_IMAGE_TAG}"

test/provisioner/provision.go

@@ -10,7 +10,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -78,40 +77,19 @@ type InstallOverlay interface {
 	Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error
 }
 
-func runCommand(command string, stdout io.Writer, stderr io.Writer, args ...string) error {
-	cmd := exec.Command(command, args...)
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-
-	fmt.Printf("Running command: %s %v\n", command, args)
-
-	if err := cmd.Run(); err != nil {
-		err = fmt.Errorf(fmt.Sprintf("Error running command: %s %v - %s", command, args, err))
-
-		log.Errorf("%v", err)
-		return err
-	}
-
-	return nil
-}
+// Waiting timeout for bringing up the pod
+const PodWaitTimeout = time.Second * 30
 
 func saveToFile(filename string, content []byte) error {
 	// Save contents to file
 	err := os.WriteFile(filename, content, 0644)
 	if err != nil {
-		return fmt.Errorf("error writing contents to file: %w", err)
+		return fmt.Errorf("writing contents to file: %w", err)
 	}
 	return nil
 }
 
 func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
-	// Clone kbs repo
-	repoURL := "https://github.com/confidential-containers/kbs"
-
-	if err := runCommand("git", os.Stdout, os.Stderr, "clone", repoURL); err != nil {
-		return nil, err
-	}
-
 	log.Info("creating key.bin")
 
 	// Create secret
@@ -120,7 +98,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	// Create the file.
 	file, err := os.Create(filePath)
 	if err != nil {
-		err = fmt.Errorf("Error creating file: %w\n", err)
+		err = fmt.Errorf("creating file: %w\n", err)
 		log.Errorf("%v", err)
 		return nil, err
 	}
@@ -129,14 +107,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	// Write the content to the file.
 	err = saveToFile(filePath, content)
 	if err != nil {
-		err = fmt.Errorf("Error writing to the file: %w\n", err)
+		err = fmt.Errorf("writing to the file: %w\n", err)
 		log.Errorf("%v", err)
 		return nil, err
 	}
 
 	k8sCnfDir, err := os.Getwd()
 	if err != nil {
-		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
+		err = fmt.Errorf("getting the current working directory: %w\n", err)
 		log.Errorf("%v", err)
 		return nil, err
 	}
@@ -147,15 +125,15 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key")
 		keyOutputFile, err := os.Create(kbsKey)
 		if err != nil {
-			err = fmt.Errorf("Error creating key file: %w\n", err)
+			err = fmt.Errorf("creating key file: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 		defer keyOutputFile.Close()
 
 		_, privateKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
-			err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err)
+			err = fmt.Errorf("generating Ed25519 key pair: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
@@ -168,15 +146,15 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		// Save private key to file
 		err = saveToFile(kbsKey, privateKeyPEM)
 		if err != nil {
-			err = fmt.Errorf("Error saving private key to file: %w\n", err)
+			err = fmt.Errorf("saving private key to file: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
 		publicKey := privateKey.Public().(ed25519.PublicKey)
 		publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey)
 		if err != nil {
-			err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err)
+			err = fmt.Errorf("generating Ed25519 public key: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
@@ -189,14 +167,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 		// Save public key to file
 		err = saveToFile(kbsCert, publicKeyPEM)
 		if err != nil {
-			err = fmt.Errorf("Error saving public key to file: %w\n", err)
+			err = fmt.Errorf("saving public key to file: %w\n", err)
 			log.Errorf("%v", err)
 			return nil, err
 		}
 
 	}
 
-	overlay, err := NewKbsInstallOverlay("kbs")
+	overlay, err := NewBaseKbsInstallOverlay("kbs")
 	if err != nil {
 		return nil, err
 	}
@@ -262,7 +240,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro
 	return overlayFunc(installDir, provider)
 }
 
-func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) {
+func NewBaseKbsInstallOverlay(installDir string) (InstallOverlay, error) {
 	log.Info("Creating kbs install overlay")
 	overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base"))
 	if err != nil {
@@ -274,6 +252,18 @@ func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) {
 	}, nil
 }
 
+func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) {
+	log.Info("Creating kbs install overlay")
+	overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/overlays"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &KbsInstallOverlay{
+		overlay: overlay,
+	}, nil
+}
+
 func (lio *KbsInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error {
 	return lio.overlay.Apply(ctx, cfg)
 }
@@ -289,7 +279,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro
 		return err
 	}
 
-	log.Infof("Updating CAA image tag with %q", props["KBS_IMAGE_TAG"])
+	log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"])
 	if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil {
 		return err
 	}
@@ -319,7 +309,7 @@ func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config)
 	serviceList := &corev1.ServiceList{}
 	err = resources.List(context.TODO(), serviceList)
 	if err != nil {
-		err = fmt.Errorf("Error listing services: %w\n", err)
+		err = fmt.Errorf("listing services: %w\n", err)
 		log.Errorf("%v", err)
 		return "", err
 	}
@@ -347,89 +337,31 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
 		return err
 	}
 
-	originalDir, err := os.Getwd()
+	// Create kustomize pointer for overlay directory with updated changes
+	tmpoverlay, err := NewKbsInstallOverlay("kbs")
 	if err != nil {
-		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
-		log.Errorf("%v", err)
 		return err
 	}
 
-	newDirectory := "kbs/kbs/config/kubernetes"
-	err = os.Chdir(newDirectory)
-	if err != nil {
-		err = fmt.Errorf("Error changing the working directory: %w\n", err)
-		log.Errorf("%v", err)
-		return err
-	}
-
-	// Replace this to use install overlay
-	cmd := exec.Command("kubectl", "apply", "-k", "overlays")
-	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
-	stdoutStderr, err := cmd.CombinedOutput()
-	log.Tracef("%v, output: %s", cmd, stdoutStderr)
-	if err != nil {
-		return err
-	}
-
-	/*
-		log.Info("Install Kbs")
-		if err := p.installOverlay.Apply(ctx, cfg); err != nil {
-			return err
-		}
-	*/
-
-	// Return to the original working directory.
-	err = os.Chdir(originalDir)
-	if err != nil {
-		err = fmt.Errorf("Error changing back to the original working directory: %w\n", err)
-		log.Errorf("%v", err)
+	log.Info("Install Kbs")
+	if err := tmpoverlay.Apply(ctx, cfg); err != nil {
 		return err
 	}
 
 	return nil
 }
 
 func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error {
-
-	originalDir, err := os.Getwd()
+	// Create kustomize pointer for overlay directory with updated changes
+	tmpoverlay, err := NewKbsInstallOverlay("kbs")
 	if err != nil {
-		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
-		log.Errorf("%v", err)
 		return err
 	}
 
-	// Remove kbs deployment
-	newDirectory := "kbs/kbs/config/kubernetes"
-	err = os.Chdir(newDirectory)
-	if err != nil {
-		err = fmt.Errorf("Error changing the working directory: %w\n", err)
-		log.Errorf("%v", err)
-		return err
-	}
-
-	log.Info("Delete Kbs deployment")
-	if err := p.installOverlay.Delete(ctx, cfg); err != nil {
-		return err
-	}
-
-	// Return to the original working directory.
-	err = os.Chdir(originalDir)
-	if err != nil {
-		err = fmt.Errorf("Error changing back to the original working directory: %w\n", err)
-		log.Errorf("%v", err)
-		return err
-	}
-
-	// remove kbs repo
-	directoryPath := "kbs"
-
-	err = os.RemoveAll(directoryPath)
-	if err != nil {
-		err = fmt.Errorf("Error deleting directory: %w\n", err)
-		log.Errorf("%v", err)
+	log.Info("Uninstall the cloud-api-adaptor")
+	if err = tmpoverlay.Delete(ctx, cfg); err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -634,7 +566,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string)
 	for _, o := range metaList {
 		obj, _ := o.(k8s.Object)
 		fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName())
-		if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil {
+		if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(PodWaitTimeout)); err != nil {
 			return err
 		}
 		fmt.Printf("pod '%s' is Ready\n", obj.GetName())