Provisioner: Add support to fetch kbs pod ip

Fixes: #1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>

install/overlays/azure/kustomization.yaml

@@ -32,13 +32,13 @@ configMapGenerator:
   # /subscriptions/<AZURE_SUBSCRIPTION_ID>/resourceGroups/<AZURE_RESOURCE_GROUP>/providers/Microsoft.Compute/images/<AZURE_IMAGE>
   - AZURE_IMAGE_ID="" #set
   - SSH_USERNAME="" #set peer pod vm admin user name
+  - AA_KBC_PARAMS="" #set
   #- DISABLECVM="" # Uncomment it if you want a generic VM
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
   #- AZURE_INSTANCE_SIZES="" # comma separated
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
   #- DISABLE_CLOUD_CONFIG="" # Uncomment if you want to enable user data for podvm
-  #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

test/e2e/main_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"fmt"
 	"context"
 	"os"
 	"testing"
@@ -14,10 +15,10 @@ import (
 )
 
 var (
-	testEnv         env.Environment
-	cloudProvider   string
-	provisioner     pv.CloudProvisioner
-	cloudAPIAdaptor *pv.CloudAPIAdaptor
+	testEnv          env.Environment
+	cloudProvider    string
+	provisioner      pv.CloudProvisioner
+	cloudAPIAdaptor  *pv.CloudAPIAdaptor
 	keyBrokerService *pv.KeyBrokerService
 )
 
@@ -71,9 +72,6 @@ func TestMain(m *testing.M) {
 	// the VPC images storage.
 	podvmImage := os.Getenv("TEST_PODVM_IMAGE")
 
-	kbsImage := os.Getenv("TEST_KBS_IMAGE")
-	kbsImageTag := os.Getenv("TEST_KBS_IMAGE_TAG")
-
 	// The TEST_PROVISION_FILE is an optional variable which specifies the path
 	// to the provision properties file. The file must have the format:
 	//
@@ -108,6 +106,8 @@ func TestMain(m *testing.M) {
 	testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 		log.Info("Do setup")
 		var err error
+		// Get properties
+		props := provisioner.GetProperties(ctx, cfg)
 
 		if shouldProvisionCluster {
 			log.Info("Cluster provisioning")
@@ -122,14 +122,24 @@ func TestMain(m *testing.M) {
 
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
+			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
+				return ctx, fmt.Errorf("kbs image not provided")
+			}
 
-			if keyBrokerService, err = pv.NewKeyBrokerService(cloudProvider); err != nil {
+			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil {
 				return ctx, err
 			}
 
-			if err = keyBrokerService.Deploy(ctx, kbsImage, kbsImageTag); err != nil {
+			if err = keyBrokerService.Deploy(ctx, props["KBS_IMAGE"], props["KBS_IMAGE_TAG"]); err != nil {
 				return ctx, err
 			}
+			var kbsPodIP string
+			if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil {
+				return ctx, err
+			}
+
+			kbsparams := "cc_kbc::http:" + kbsPodIP + ":8080"
+			props["AA_KBC_PARAMS"] = kbsparams
 		}
 
 		if podvmImage != "" {
@@ -143,7 +153,7 @@ func TestMain(m *testing.M) {
 			return ctx, err
 		}
 		log.Info("Deploy the Cloud API Adaptor")
-		if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil {
+		if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil {
 			return ctx, err
 		}
 		return ctx, nil

test/provisioner/provision.go

@@ -51,10 +51,7 @@ type CloudAPIAdaptor struct {
 	runtimeClass         *nodev1.RuntimeClass // The Kata Containers runtimeclass
 }
 
-type KeyBrokerService struct {
-        cloudProvider        string               // Cloud provider
-}
-
+type KeyBrokerService struct {}
 
 type newInstallOverlayFunc func(installDir string) (InstallOverlay, error)
 
@@ -70,7 +67,7 @@ type InstallOverlay interface {
 	Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error
 }
 
-func NewKeyBrokerService(provider string) (*KeyBrokerService, error) {
+func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
 	// Clone kbs repo
 	repoURL := "https://github.com/confidential-containers/kbs"
 	cmd := exec.Command("git", "clone", repoURL)
@@ -84,7 +81,7 @@ func NewKeyBrokerService(provider string) (*KeyBrokerService, error) {
 	}
 
 	// Create secret
-	content := []byte("This is my super secret")
+	content := []byte("This is my cluster name: " + clusterName)
 	filePath := "kbs/config/kubernetes/overlays/key.bin"
 	// Create the file.
 	file, err := os.Create(filePath)
@@ -101,9 +98,7 @@ func NewKeyBrokerService(provider string) (*KeyBrokerService, error) {
 		return nil, err
 	}
 
-	return &KeyBrokerService{
-		cloudProvider:        provider,
-	}, nil
+	return &KeyBrokerService{}, nil
 }
 
 func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, error) {
@@ -187,6 +182,47 @@ func UpdateKbsKustomizationFile(imagePath string, imageTag string) error {
 
 }
 
+func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) {
+	client, err := cfg.NewClient()
+	if err != nil {
+		return "", err
+	}
+
+	namespace := "coco-tenant"
+	deploymentName := "kbs"
+
+	err = AllPodsRunning(ctx, cfg, namespace)
+	if err != nil {
+		fmt.Printf("All pods are not running: %v\n", err)
+		return "", err
+	}
+
+	resources := client.Resources(namespace)
+
+	podList := &corev1.PodList{}
+	err = resources.List(context.TODO(), podList)
+	if err != nil {
+		fmt.Printf("Error listing pods: %v\n", err)
+		return "", err
+	}
+
+	var matchingPod *corev1.Pod
+	for i := range podList.Items {
+		pod := &podList.Items[i]
+		if pod.Labels["app"] == deploymentName {
+			matchingPod = pod
+			break
+		}
+	}
+
+	if matchingPod == nil {
+		return "", fmt.Errorf("No pod with label selector found")
+	}
+
+	fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP)
+	return matchingPod.Status.PodIP, nil
+}
+
 func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTag string) error {
 	originalDir, err := os.Getwd()
 	if err != nil {
@@ -220,7 +256,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, imagePath string, imageTa
 	keyFile := filepath.Join(k8sCnfDir, "overlays/key.bin")
 	if _, err := os.Stat(keyFile); os.IsNotExist(err) {
 		fmt.Println("key.bin file does not exist")
-		//return err
+		return err
 	}
 
 	kbsCert := filepath.Join(k8sCnfDir, "base/kbs.pem")

test/provisioner/provision_azure.go

@@ -361,6 +361,8 @@ func (p *AzureCloudProvisioner) GetProperties(ctx context.Context, cfg *envconf.
 		"AZURE_IMAGE_ID":        AzureProps.ImageID,
 		"AZURE_SUBNET_ID":       AzureProps.SubnetID,
 		"AZURE_INSTANCE_SIZE":   AzureProps.InstanceSize,
+		"KBS_IMAGE":             AzureProps.KbsImage,
+		"KBS_IMAGE_TAG":         AzureProps.KbsImageTag,
 	}
 
 	return props
@@ -374,7 +376,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex
 
 func isAzureKustomizeConfigMapKey(key string) bool {
 	switch key {
-	case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME":
+	case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "AA_KBC_PARAMS":
 		return true
 	default:
 		return false

test/provisioner/provision_azure.properties

@@ -12,3 +12,5 @@ AZURE_IMAGE_ID=""
 SSH_USERNAME=""
 AZURE_CLI_AUTH="false"
 IS_CI_MANAGED_CLUSTER="false"
+KBS_IMAGE=""
+KBS_IMAGE_TAG="latest"

test/provisioner/provision_azure_initializer.go

@@ -32,6 +32,8 @@ type AzureProperties struct {
 	ManagedIdentityName string
 	IsCIManaged         bool
 	CaaImage            string
+	KbsImage            string
+	KbsImageTag         string
 
 	InstanceSize string
 	NodeName     string
@@ -63,6 +65,8 @@ func initAzureProperties(properties map[string]string) error {
 		SshUserName:         properties["SSH_USERNAME"],
 		ManagedIdentityName: properties["MANAGED_IDENTITY_NAME"],
 		CaaImage:            properties["CAA_IMAGE"],
+		KbsImage:            properties["KBS_IMAGE"],
+		KbsImageTag:         properties["KBS_IMAGE_TAG"],
 	}
 
 	CIManagedStr := properties["IS_CI_MANAGED_CLUSTER"]