Skip to content

Commit

Permalink
Provisioner: Add support to fetch kbs service ip
Browse files Browse the repository at this point in the history 
Fixes: #1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>
  • Loading branch information
@kartikjoshi21
kartikjoshi21 committed Jan 23, 2024
1 parent f82c7a6 commit 8ad754e
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 45 deletions.
26 changes: 15 additions & 11 deletions test/e2e/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import (
)

var (
testEnv env.Environment
cloudProvider string
provisioner pv.CloudProvisioner
testEnv env.Environment
cloudProvider string
provisioner pv.CloudProvisioner
keyBrokerService *pv.KeyBrokerService
)

Expand Down Expand Up @@ -96,9 +96,9 @@ func TestMain(m *testing.M) {
}

// The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA
shouldDeployKbs := false
if os.Getenv("DEPLOY_KBS") == "yes" {
shouldDeployKbs = true
shouldDeployKbs := true
if os.Getenv("DEPLOY_KBS") != "yes" {
shouldDeployKbs = false
}

if !shouldProvisionCluster {
Expand All @@ -117,7 +117,6 @@ func TestMain(m *testing.M) {
log.Info("Do setup")
var err error
// Get properties
props := provisioner.GetProperties(ctx, cfg)

if shouldProvisionCluster {
log.Info("Cluster provisioning")
Expand All @@ -130,6 +129,8 @@ func TestMain(m *testing.M) {
}
}

props := provisioner.GetProperties(ctx, cfg)
var kbsparams string
if shouldDeployKbs {
log.Info("Deploying kbs")
if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
Expand All @@ -144,12 +145,12 @@ func TestMain(m *testing.M) {
return ctx, err
}
var kbsPodIP string
if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil {
if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
return ctx, err
}

kbsparams := "cc_kbc::http:" + kbsPodIP + ":8080"
props["AA_KBC_PARAMS"] = kbsparams
kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080"
log.Infof("KBS PARAMS%s:", kbsparams)
}

if podvmImage != "" {
Expand All @@ -165,8 +166,11 @@ func TestMain(m *testing.M) {
if cloudAPIAdaptor, err = pv.NewCloudAPIAdaptor(cloudProvider, relativeInstallDirectory); err != nil {
return ctx, err
}

props = provisioner.GetProperties(ctx, cfg)
props["AA_KBC_PARAMS"] = kbsparams
log.Info("Deploy the Cloud API Adaptor")
if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil {
if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil {
return ctx, err
}
}
Expand Down
2 changes: 1 addition & 1 deletion test/provisioner/azure/provision_azure.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ SSH_USERNAME=""
AZURE_CLI_AUTH="false"
IS_CI_MANAGED_CLUSTER="false"
IS_SELF_MANAGED_CLUSTER="false"
KBS_IMAGE="quay.io/karikjoshi21/kbs/coco-as-21705eb"
KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs"
KBS_IMAGE_TAG="latest"
78 changes: 45 additions & 33 deletions test/provisioner/provision.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {

// Create secret
content := []byte("This is my cluster name: " + clusterName)
filePath := "kbs/config/kubernetes/overlays/key.bin"
filePath := "kbs/kbs/config/kubernetes/overlays/key.bin"
// Create the file.
file, err := os.Create(filePath)
if err != nil {
Expand All @@ -129,16 +129,16 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
}
fmt.Println(k8sCnfDir)

keyFile := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/overlays/key.bin")
keyFile := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/overlays/key.bin")
if _, err := os.Stat(keyFile); os.IsNotExist(err) {
err = fmt.Errorf("key.bin file does not exist")
log.Errorf("%v", err)
return nil, err
}

kbsCert := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.pem")
kbsCert := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.pem")
if _, err := os.Stat(kbsCert); os.IsNotExist(err) {
kbsKey := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.key")
kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key")
keyOutputFile, err := os.Create(kbsKey)
if err != nil {
err = fmt.Errorf("Error creating key file: %w\n", err)
Expand Down Expand Up @@ -224,7 +224,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro

func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) {
log.Info("Creating kbs install overlay")
overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "config/kubernetes/base"))
overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base"))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -257,7 +257,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro
return nil
}

func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) {
func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) (string, error) {
client, err := cfg.NewClient()
if err != nil {
return "", err
Expand All @@ -275,29 +275,30 @@ func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config)

resources := client.Resources(namespace)

podList := &corev1.PodList{}
err = resources.List(context.TODO(), podList)
if err != nil {
err = fmt.Errorf("Error listing pods: %w\n", err)
log.Errorf("%v", err)
return "", err
}

var matchingPod *corev1.Pod
for i := range podList.Items {
pod := &podList.Items[i]
if pod.Labels["app"] == deploymentName {
matchingPod = pod
break
}
}

if matchingPod == nil {
return "", fmt.Errorf("No pod with label selector found")
}

fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP)
return matchingPod.Status.PodIP, nil
// Get the service associated with the deployment
serviceList := &corev1.ServiceList{}
err = resources.List(context.TODO(), serviceList)
if err != nil {
err = fmt.Errorf("Error listing services: %w\n", err)
log.Errorf("%v", err)
return "", err
}

var matchingService *corev1.Service
for i := range serviceList.Items {
service := &serviceList.Items[i]
if service.Name == deploymentName {
matchingService = service
break
}
}

if matchingService == nil {
return "", fmt.Errorf("No service with label selector found")
}

fmt.Printf("Service IP: %s\n", matchingService.Spec.ClusterIP)
return matchingService.Spec.ClusterIP, nil
}

func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error {
Expand All @@ -313,18 +314,29 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
return err
}

newDirectory := "kbs/config/kubernetes/overlays"
newDirectory := "kbs/kbs/config/kubernetes"
err = os.Chdir(newDirectory)
if err != nil {
err = fmt.Errorf("Error changing the working directory: %w\n", err)
log.Errorf("%v", err)
return err
}

// Replace this to use install overlay
cmd := exec.Command("kubectl", "apply", "-k", "overlays")
cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
stdoutStderr, err := cmd.CombinedOutput()
log.Tracef("%v, output: %s", cmd, stdoutStderr)
if err != nil {
return err
}

/*
log.Info("Install Kbs")
if err := p.installOverlay.Apply(ctx, cfg); err != nil {
return err
}
*/

// Return to the original working directory.
err = os.Chdir(originalDir)
Expand All @@ -347,15 +359,15 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro
}

// Remove kbs deployment
newDirectory := "kbs/config/kubernetes/overlays"
newDirectory := "kbs/kbs/config/kubernetes"
err = os.Chdir(newDirectory)
if err != nil {
err = fmt.Errorf("Error changing the working directory: %w\n", err)
log.Errorf("%v", err)
return err
}

log.Info("Install Kbs")
log.Info("Delete Kbs deployment")
if err := p.installOverlay.Delete(ctx, cfg); err != nil {
return err
}
Expand Down Expand Up @@ -582,7 +594,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string)
for _, o := range metaList {
obj, _ := o.(k8s.Object)
fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName())
if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*6)); err != nil {
if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil {
return err
}
fmt.Printf("pod '%s' is Ready\n", obj.GetName())
Expand Down

0 comments on commit 8ad754e

Please sign in to comment.