Provisioner: Add support to fetch kbs pod ip

Fixes: #1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>

install/overlays/azure/kustomization.yaml

@@ -32,13 +32,13 @@ configMapGenerator:
   # /subscriptions/<AZURE_SUBSCRIPTION_ID>/resourceGroups/<AZURE_RESOURCE_GROUP>/providers/Microsoft.Compute/images/<AZURE_IMAGE>
   - AZURE_IMAGE_ID="" #set
   - SSH_USERNAME="" #set peer pod vm admin user name
+  - AA_KBC_PARAMS="" #set
   #- DISABLECVM="" # Uncomment it if you want a generic VM
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
   #- AZURE_INSTANCE_SIZES="" # comma separated
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
   #- DISABLE_CLOUD_CONFIG="" # Uncomment if you want to enable user data for podvm
-  #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

test/e2e/main_test.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"testing"
 
@@ -14,10 +15,10 @@ import (
 )
 
 var (
-	testEnv         env.Environment
-	cloudProvider   string
-	provisioner     pv.CloudProvisioner
-	cloudAPIAdaptor *pv.CloudAPIAdaptor
+	testEnv          env.Environment
+	cloudProvider    string
+	provisioner      pv.CloudProvisioner
+	cloudAPIAdaptor  *pv.CloudAPIAdaptor
 	keyBrokerService *pv.KeyBrokerService
 )
 
@@ -71,9 +72,6 @@ func TestMain(m *testing.M) {
 	// the VPC images storage.
 	podvmImage := os.Getenv("TEST_PODVM_IMAGE")
 
-	kbsImage := os.Getenv("TEST_KBS_IMAGE")
-	kbsImageTag := os.Getenv("TEST_KBS_IMAGE_TAG")
-
 	// The TEST_PROVISION_FILE is an optional variable which specifies the path
 	// to the provision properties file. The file must have the format:
 	//
@@ -108,6 +106,8 @@ func TestMain(m *testing.M) {
 	testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 		log.Info("Do setup")
 		var err error
+		// Get properties
+		props := provisioner.GetProperties(ctx, cfg)
 
 		if shouldProvisionCluster {
 			log.Info("Cluster provisioning")
@@ -122,14 +122,24 @@ func TestMain(m *testing.M) {
 
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
+			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
+				return ctx, fmt.Errorf("kbs image not provided")
+			}
 
-			if keyBrokerService, err = pv.NewKeyBrokerService(cloudProvider); err != nil {
+			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil {
 				return ctx, err
 			}
 
-			if err = keyBrokerService.Deploy(ctx, kbsImage, kbsImageTag); err != nil {
+			if err = keyBrokerService.Deploy(ctx, props["KBS_IMAGE"], props["KBS_IMAGE_TAG"]); err != nil {
 				return ctx, err
 			}
+			var kbsPodIP string
+			if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil {
+				return ctx, err
+			}
+
+			kbsparams := "cc_kbc::http:" + kbsPodIP + ":8080"
+			props["AA_KBC_PARAMS"] = kbsparams
 		}
 
 		if podvmImage != "" {
@@ -143,7 +153,7 @@ func TestMain(m *testing.M) {
 			return ctx, err
 		}
 		log.Info("Deploy the Cloud API Adaptor")
-		if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil {
+		if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil {
 			return ctx, err
 		}
 		return ctx, nil

test/provisioner/provision.go

@@ -6,12 +6,12 @@ package provisioner
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
-	"time"
-	"strings"
 	"path/filepath"
-	"io/ioutil"
+	"strings"
+	"time"
 
 	"github.com/BurntSushi/toml"
 	log "github.com/sirupsen/logrus"
@@ -51,10 +51,7 @@ type CloudAPIAdaptor struct {
 	runtimeClass         *nodev1.RuntimeClass // The Kata Containers runtimeclass
 }
 
-type KeyBrokerService struct {
-        cloudProvider        string               // Cloud provider
-}
-
+type KeyBrokerService struct{}
 
 type newInstallOverlayFunc func(installDir string) (InstallOverlay, error)
 
@@ -70,40 +67,50 @@ type InstallOverlay interface {
 	Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error
 }
 
-func NewKeyBrokerService(provider string) (*KeyBrokerService, error) {
-	// Clone kbs repo
-	repoURL := "https://github.com/confidential-containers/kbs"
-	cmd := exec.Command("git", "clone", repoURL)
+func runCommand(command string, args ...string) error {
+	cmd := exec.Command(command, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
+	fmt.Printf("Running command: %s %v\n", command, args)
 
-	err := cmd.Run()
-	if err != nil {
-		fmt.Printf("Error running git clone: %v\n", err)
+	if err := cmd.Run(); err != nil {
+		err = fmt.Errorf(fmt.Sprintf("Error running command: %s %v - %s", command, args, err))
+		log.Errorf("%v", err)
+		return err
+	}
+
+	return nil
+}
+
+func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
+	// Clone kbs repo
+	repoURL := "https://github.com/confidential-containers/kbs"
+
+	if err := runCommand("git", "clone", repoURL); err != nil {
 		return nil, err
 	}
 
 	// Create secret
-	content := []byte("This is my super secret")
+	content := []byte("This is my cluster name: " + clusterName)
 	filePath := "kbs/config/kubernetes/overlays/key.bin"
 	// Create the file.
 	file, err := os.Create(filePath)
 	if err != nil {
-		fmt.Printf("Error creating file: %v\n", err)
+		err = fmt.Errorf("Error creating file: %w\n", err)
+		log.Errorf("%v", err)
 		return nil, err
 	}
 	defer file.Close()
 
 	// Write the content to the file.
 	_, err = file.Write(content)
 	if err != nil {
-		fmt.Printf("Error writing to file: %v\n", err)
+		err = fmt.Errorf("Error writing to the file: %w\n", err)
+		log.Errorf("%v", err)
 		return nil, err
 	}
 
-	return &KeyBrokerService{
-		cloudProvider:        provider,
-	}, nil
+	return &KeyBrokerService{}, nil
 }
 
 func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, error) {
@@ -164,7 +171,8 @@ func UpdateKbsKustomizationFile(imagePath string, imageTag string) error {
 	filePath := "base/kustomization.yaml"
 	content, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		fmt.Printf("Error reading kustomization file: %v\n", err)
+		err = fmt.Errorf("Error reading kustomization file: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
@@ -178,7 +186,8 @@ func UpdateKbsKustomizationFile(imagePath string, imageTag string) error {
 	// Write the updated content back to the same file.
 	err = ioutil.WriteFile(filePath, []byte(kustomizationContent), 0644)
 	if err != nil {
-		fmt.Printf("Error writing to kustomization file: %v\n", err)
+		err = fmt.Errorf("Error writing to kustomization file: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
@@ -187,82 +196,119 @@ func UpdateKbsKustomizationFile(imagePath string, imageTag string) error {
 
 }
 
+func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) {
+	client, err := cfg.NewClient()
+	if err != nil {
+		return "", err
+	}
+
+	namespace := "coco-tenant"
+	deploymentName := "kbs"
+
+	err = AllPodsRunning(ctx, cfg, namespace)
+	if err != nil {
+		err = fmt.Errorf("All pods are not running: %w\n", err)
+		log.Errorf("%v", err)
+		return "", err
+	}
+
+	resources := client.Resources(namespace)
+
+	podList := &corev1.PodList{}
+	err = resources.List(context.TODO(), podList)
+	if err != nil {
+		err = fmt.Errorf("Error listing pods: %w\n", err)
+		log.Errorf("%v", err)
+		return "", err
+	}
+
+	var matchingPod *corev1.Pod
+	for i := range podList.Items {
+		pod := &podList.Items[i]
+		if pod.Labels["app"] == deploymentName {
+			matchingPod = pod
+			break
+		}
+	}
+
+	if matchingPod == nil {
+		return "", fmt.Errorf("No pod with label selector found")
+	}
+
+	fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP)
+	return matchingPod.Status.PodIP, nil
+}
+
 func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTag string) error {
 	originalDir, err := os.Getwd()
 	if err != nil {
-		fmt.Printf("Error getting the current working directory: %v\n", err)
+		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
 	// jump to kbs kubernetes config directory
 	newDirectory := "kbs/config/kubernetes/"
 	err = os.Chdir(newDirectory)
 	if err != nil {
-		fmt.Printf("Error changing the working directory: %v\n", err)
+		err = fmt.Errorf("Error changing the working directory: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
 	// Note: Use kustomize overlay to update this
 	err = UpdateKbsKustomizationFile(imagePath, imageTag)
 	if err != nil {
-		fmt.Printf("Error updating kustomization file: %v\n", err)
+		err = fmt.Errorf("Error updating kustomization file: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
 	// Deploy kbs
 	k8sCnfDir, err := os.Getwd()
 	if err != nil {
-		fmt.Printf("Error getting the current working directory: %v\n", err)
+		err = fmt.Errorf("Error getting the current working directory: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 	fmt.Println(k8sCnfDir)
 
 	keyFile := filepath.Join(k8sCnfDir, "overlays/key.bin")
 	if _, err := os.Stat(keyFile); os.IsNotExist(err) {
-		fmt.Println("key.bin file does not exist")
-		//return err
+		err = fmt.Errorf("key.bin file does not exist")
+		log.Errorf("%v", err)
+		return err
 	}
 
 	kbsCert := filepath.Join(k8sCnfDir, "base/kbs.pem")
 	if _, err := os.Stat(kbsCert); os.IsNotExist(err) {
 		kbsKey := filepath.Join(k8sCnfDir, "base/kbs.key")
 		keyOutputFile, err := os.Create(kbsKey)
 		if err != nil {
-			fmt.Printf("Error creating key file: %v\n", err)
-			os.Exit(1)
+			err = fmt.Errorf("Error creating key file: %w\n", err)
+			log.Errorf("%v", err)
+			return err
 		}
 		defer keyOutputFile.Close()
 
-		opensslGenPKeyCmd := exec.Command("openssl", "genpkey", "-algorithm", "ed25519")
-		opensslGenPKeyCmd.Stdout = keyOutputFile
-		opensslGenPKeyCmd.Stderr = os.Stderr
-		fmt.Printf("Running command: %v\n", opensslGenPKeyCmd.Args)
-		if err := opensslGenPKeyCmd.Run(); err != nil {
-			fmt.Printf("Error generating key: %v\n", err)
+		if err := runCommand("openssl", "genpkey", "-algorithm", "ed25519"); err != nil {
 			return err
 		}
 
-		opensslPKeyCmd := exec.Command("openssl", "pkey", "-in", kbsKey, "-pubout", "-out", kbsCert)
-		opensslPKeyCmd.Stdout = os.Stdout
-		opensslPKeyCmd.Stderr = os.Stderr
-		if err := opensslPKeyCmd.Run(); err != nil {
-			fmt.Printf("Error creating kbs.pem: %v\n", err)
+		if err := runCommand("openssl", "pkey", "-in", kbsKey, "-pubout", "-out", kbsCert); err != nil {
 			return err
 		}
 	}
 
-	kubectlApplyCmd := exec.Command("kubectl", "apply", "-k", k8sCnfDir+"/overlays")
-	kubectlApplyCmd.Stdout = os.Stdout
-	kubectlApplyCmd.Stderr = os.Stderr
-	if err := kubectlApplyCmd.Run(); err != nil {
-		fmt.Printf("Error running 'kubectl apply': %v\n", err)
+	if err := runCommand("kubectl", "apply", "-k", k8sCnfDir+"/overlays"); err != nil {
 		return err
 	}
 
 	// Return to the original working directory.
 	err = os.Chdir(originalDir)
 	if err != nil {
-		fmt.Printf("Error changing back to the original working directory: %v\n", err)
+		err = fmt.Errorf("Error changing back to the original working directory: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
@@ -271,7 +317,8 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTa
 
 	err = os.RemoveAll(directoryPath)
 	if err != nil {
-		fmt.Printf("Error deleting directory: %v\n", err)
+		err = fmt.Errorf("Error deleting directory: %w\n", err)
+		log.Errorf("%v", err)
 		return err
 	}
 
@@ -281,20 +328,14 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTa
 func (p *KeyBrokerService) Delete(ctx context.Context) error {
 	// Remove kbs deployment
 	k8sCnfDir := "kbs/config/kubernetes"
-	kubectlDeleteCmd := exec.Command("kubectl", "delete", "-k", k8sCnfDir+"/overlays")
-	kubectlDeleteCmd.Stdout = os.Stdout
-	kubectlDeleteCmd.Stderr = os.Stderr
 
-	err := kubectlDeleteCmd.Run()
-	if err != nil {
-		fmt.Printf("Error running 'kubectl delete': %v\n", err)
+	if err := runCommand("kubectl", "delete", "-k", k8sCnfDir+"/overlays"); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-
 // Deletes the peer pods installation including the controller manager.
 func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error {
 	client, err := cfg.NewClient()

test/provisioner/provision_azure.go

@@ -361,6 +361,8 @@ func (p *AzureCloudProvisioner) GetProperties(ctx context.Context, cfg *envconf.
 		"AZURE_IMAGE_ID":        AzureProps.ImageID,
 		"AZURE_SUBNET_ID":       AzureProps.SubnetID,
 		"AZURE_INSTANCE_SIZE":   AzureProps.InstanceSize,
+		"KBS_IMAGE":             AzureProps.KbsImage,
+		"KBS_IMAGE_TAG":         AzureProps.KbsImageTag,
 	}
 
 	return props
@@ -374,7 +376,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex
 
 func isAzureKustomizeConfigMapKey(key string) bool {
 	switch key {
-	case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME":
+	case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "AA_KBC_PARAMS":
 		return true
 	default:
 		return false