Provisioner: Add support to fetch kbs service ip

Fixes: confidential-containers#1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>

install/overlays/azure/kustomization.yaml

@@ -32,6 +32,7 @@ configMapGenerator:
   # /subscriptions/<AZURE_SUBSCRIPTION_ID>/resourceGroups/<AZURE_RESOURCE_GROUP>/providers/Microsoft.Compute/images/<AZURE_IMAGE>
   - AZURE_IMAGE_ID="" #set
   - SSH_USERNAME="" #set peer pod vm admin user name
+  - AA_KBC_PARAMS="" #set
   #- DISABLECVM="" # Uncomment it if you want a generic VM
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789

test/e2e/main_test.go

@@ -5,6 +5,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"testing"
 
@@ -17,12 +18,9 @@ import (
 )
 
 var (
-	testEnv       env.Environment
-	cloudProvider string
-	provisioner   pv.CloudProvisioner
-	testEnv         env.Environment
-	cloudProvider   string
-	provisioner     pv.CloudProvisioner
+	testEnv          env.Environment
+	cloudProvider    string
+	provisioner      pv.CloudProvisioner
 	keyBrokerService *pv.KeyBrokerService
 )
 
@@ -84,9 +82,6 @@ func TestMain(m *testing.M) {
 	// the VPC images storage.
 	podvmImage := os.Getenv("TEST_PODVM_IMAGE")
 
-	kbsImage := os.Getenv("TEST_KBS_IMAGE")
-	kbsImageTag := os.Getenv("TEST_KBS_IMAGE_TAG")
-
 	// The TEST_PROVISION_FILE is an optional variable which specifies the path
 	// to the provision properties file. The file must have the format:
 	//
@@ -101,9 +96,9 @@ func TestMain(m *testing.M) {
 	}
 
 	// The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA
-	shouldDeployKbs := false
-	if os.Getenv("DEPLOY_KBS") == "yes" {
-		shouldDeployKbs = true
+	shouldDeployKbs := true
+	if os.Getenv("DEPLOY_KBS") != "yes" {
+		shouldDeployKbs = false
 	}
 
 	if !shouldProvisionCluster {
@@ -121,6 +116,7 @@ func TestMain(m *testing.M) {
 	testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 		log.Info("Do setup")
 		var err error
+		// Get properties
 
 		if shouldProvisionCluster {
 			log.Info("Cluster provisioning")
@@ -133,16 +129,28 @@ func TestMain(m *testing.M) {
 			}
 		}
 
+		props := provisioner.GetProperties(ctx, cfg)
+		var kbsparams string
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
+			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
+				return ctx, fmt.Errorf("kbs image not provided")
+			}
 
-			if keyBrokerService, err = pv.NewKeyBrokerService(cloudProvider); err != nil {
+			if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil {
 				return ctx, err
 			}
 
-			if err = keyBrokerService.Deploy(ctx, kbsImage, kbsImageTag); err != nil {
+			if err = keyBrokerService.Deploy(ctx, cfg, props); err != nil {
+				return ctx, err
+			}
+			var kbsPodIP string
+			if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
 				return ctx, err
 			}
+
+			kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080"
+			log.Infof("KBS PARAMS%s:", kbsparams)
 		}
 
 		if podvmImage != "" {
@@ -158,8 +166,11 @@ func TestMain(m *testing.M) {
 			if cloudAPIAdaptor, err = pv.NewCloudAPIAdaptor(cloudProvider, relativeInstallDirectory); err != nil {
 				return ctx, err
 			}
+
+			props = provisioner.GetProperties(ctx, cfg)
+			props["AA_KBC_PARAMS"] = kbsparams
 			log.Info("Deploy the Cloud API Adaptor")
-			if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil {
+			if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil {
 				return ctx, err
 			}
 		}
@@ -199,7 +210,7 @@ func TestMain(m *testing.M) {
 		}
 
 		if shouldDeployKbs {
-			if err = keyBrokerService.Delete(ctx); err != nil {
+			if err = keyBrokerService.Delete(ctx, cfg); err != nil {
 				return ctx, err
 			}
 		}

test/provisioner/azure/provision_azure.properties

@@ -13,3 +13,5 @@ SSH_USERNAME=""
 AZURE_CLI_AUTH="false"
 IS_CI_MANAGED_CLUSTER="false"
 IS_SELF_MANAGED_CLUSTER="false"
+KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs"
+KBS_IMAGE_TAG="latest"

test/provisioner/azure/provision_common.go

@@ -360,6 +360,8 @@ func getPropertiesImpl() map[string]string {
 		"AZURE_IMAGE_ID":        AzureProps.ImageID,
 		"AZURE_SUBNET_ID":       AzureProps.SubnetID,
 		"AZURE_INSTANCE_SIZE":   AzureProps.InstanceSize,
+		"KBS_IMAGE":             AzureProps.KbsImage,
+		"KBS_IMAGE_TAG":         AzureProps.KbsImageTag,
 	}
 
 	return props
@@ -378,7 +380,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex
 
 func isAzureKustomizeConfigMapKey(key string) bool {
 	switch key {
-	case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME":
+	case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "AA_KBC_PARAMS":
 		return true
 	default:
 		return false

test/provisioner/azure/provision_initializer.go

@@ -33,6 +33,8 @@ type AzureProperties struct {
 	IsCIManaged         bool
 	CaaImage            string
 	IsSelfManaged       bool
+	KbsImage            string
+	KbsImageTag         string
 
 	InstanceSize string
 	NodeName     string
@@ -64,6 +66,8 @@ func initAzureProperties(properties map[string]string) error {
 		SshUserName:         properties["SSH_USERNAME"],
 		ManagedIdentityName: properties["MANAGED_IDENTITY_NAME"],
 		CaaImage:            properties["CAA_IMAGE"],
+		KbsImage:            properties["KBS_IMAGE"],
+		KbsImageTag:         properties["KBS_IMAGE_TAG"],
 	}
 
 	CIManagedStr := properties["IS_CI_MANAGED_CLUSTER"]