Skip to content

Commit

Permalink
Provisioner: Add support to fetch kbs service ip
Browse files Browse the repository at this point in the history 
Fixes: confidential-containers#1471
Signed-off-by: Kartik Joshi <kartikjoshi@microsoft.com>
  • Loading branch information
@kartikjoshi21
kartikjoshi21 committed Feb 12, 2024
1 parent c9fae38 commit 2005f51
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 45 deletions.
26 changes: 15 additions & 11 deletions test/e2e/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import (
)

var (
testEnv env.Environment
cloudProvider string
provisioner pv.CloudProvisioner
testEnv env.Environment
cloudProvider string
provisioner pv.CloudProvisioner
keyBrokerService *pv.KeyBrokerService
)

Expand Down Expand Up @@ -96,9 +96,9 @@ func TestMain(m *testing.M) {
}

// The DEPLOY_KBS is exported then provisioner will install kbs before installing CAA
shouldDeployKbs := false
if os.Getenv("DEPLOY_KBS") == "yes" {
shouldDeployKbs = true
shouldDeployKbs := true
if os.Getenv("DEPLOY_KBS") != "yes" {
shouldDeployKbs = false
}

if !shouldProvisionCluster {
Expand All @@ -117,7 +117,6 @@ func TestMain(m *testing.M) {
log.Info("Do setup")
var err error
// Get properties
props := provisioner.GetProperties(ctx, cfg)

if shouldProvisionCluster {
log.Info("Cluster provisioning")
Expand All @@ -130,6 +129,8 @@ func TestMain(m *testing.M) {
}
}

props := provisioner.GetProperties(ctx, cfg)
var kbsparams string
if shouldDeployKbs {
log.Info("Deploying kbs")
if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
Expand All @@ -144,12 +145,12 @@ func TestMain(m *testing.M) {
return ctx, err
}
var kbsPodIP string
if kbsPodIP, err = keyBrokerService.GetKbsPodIP(ctx, cfg); err != nil {
if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil {
return ctx, err
}

kbsparams := "cc_kbc::http:" + kbsPodIP + ":8080"
props["AA_KBC_PARAMS"] = kbsparams
kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080"
log.Infof("KBS PARAMS%s:", kbsparams)
}

if podvmImage != "" {
Expand All @@ -165,8 +166,11 @@ func TestMain(m *testing.M) {
if cloudAPIAdaptor, err = pv.NewCloudAPIAdaptor(cloudProvider, relativeInstallDirectory); err != nil {
return ctx, err
}

props = provisioner.GetProperties(ctx, cfg)
props["AA_KBC_PARAMS"] = kbsparams
log.Info("Deploy the Cloud API Adaptor")
if err = cloudAPIAdaptor.Deploy(ctx, cfg, provisioner.GetProperties(ctx, cfg)); err != nil {
if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil {
return ctx, err
}
}
Expand Down
2 changes: 1 addition & 1 deletion test/provisioner/azure/provision_azure.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ SSH_USERNAME=""
AZURE_CLI_AUTH="false"
IS_CI_MANAGED_CLUSTER="false"
IS_SELF_MANAGED_CLUSTER="false"
KBS_IMAGE="quay.io/karikjoshi21/kbs/coco-as-21705eb"
KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs"
KBS_IMAGE_TAG="latest"
118 changes: 85 additions & 33 deletions test/provisioner/provision.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ package provisioner

import (
"context"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"encoding/pem"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -91,6 +95,15 @@ func runCommand(command string, stdout io.Writer, stderr io.Writer, args ...stri
return nil
}

func saveToFile(filename string, content []byte) error {
// Save contents to file
err := os.WriteFile(filename, content, 0644)
if err != nil {
return fmt.Errorf("error writing contents to file: %w", err)
}
return nil
}

func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
// Clone kbs repo
repoURL := "https://github.com/confidential-containers/kbs"
Expand All @@ -103,7 +116,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {

// Create secret
content := []byte("This is my cluster name: " + clusterName)
filePath := "kbs/config/kubernetes/overlays/key.bin"
filePath := "kbs/kbs/config/kubernetes/overlays/key.bin"
// Create the file.
file, err := os.Create(filePath)
if err != nil {
Expand All @@ -114,7 +127,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
defer file.Close()

// Write the content to the file.
_, err = file.Write(content)
err = saveToFile(filePath, content)
if err != nil {
err = fmt.Errorf("Error writing to the file: %w\n", err)
log.Errorf("%v", err)
Expand All @@ -129,16 +142,9 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
}
fmt.Println(k8sCnfDir)

keyFile := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/overlays/key.bin")
if _, err := os.Stat(keyFile); os.IsNotExist(err) {
err = fmt.Errorf("key.bin file does not exist")
log.Errorf("%v", err)
return nil, err
}

kbsCert := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.pem")
kbsCert := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.pem")
if _, err := os.Stat(kbsCert); os.IsNotExist(err) {
kbsKey := filepath.Join(k8sCnfDir, "kbs/config/kubernetes/base/kbs.key")
kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key")
keyOutputFile, err := os.Create(kbsKey)
if err != nil {
err = fmt.Errorf("Error creating key file: %w\n", err)
Expand All @@ -147,13 +153,47 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) {
}
defer keyOutputFile.Close()

if err := runCommand("openssl", keyOutputFile, os.Stderr, "genpkey", "-algorithm", "ed25519"); err != nil {
_, privateKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err)
log.Errorf("%v", err)
return nil, err
}

privateKeyPEM := pem.EncodeToMemory(&pem.Block{
Type: "PRIVATE KEY",
Bytes: privateKey,
})

// Save private key to file
err = saveToFile(kbsKey, privateKeyPEM)
if err != nil {
err = fmt.Errorf("Error saving private key to file: %w\n", err)
log.Errorf("%v", err)
return nil, err
}

publicKey := privateKey.Public().(ed25519.PublicKey)
publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey)
if err != nil {
err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err)
log.Errorf("%v", err)
return nil, err
}

if err := runCommand("openssl", os.Stdout, os.Stderr, "pkey", "-in", kbsKey, "-pubout", "-out", kbsCert); err != nil {
publicKeyPEM := pem.EncodeToMemory(&pem.Block{
Type: "PUBLIC KEY",
Bytes: publicKeyX509,
})

// Save public key to file
err = saveToFile(kbsCert, publicKeyPEM)
if err != nil {
err = fmt.Errorf("Error saving public key to file: %w\n", err)
log.Errorf("%v", err)
return nil, err
}

}

overlay, err := NewKbsInstallOverlay("kbs")
Expand Down Expand Up @@ -224,7 +264,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro

func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) {
log.Info("Creating kbs install overlay")
overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "config/kubernetes/base"))
overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base"))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -257,7 +297,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro
return nil
}

func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config) (string, error) {
func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) (string, error) {
client, err := cfg.NewClient()
if err != nil {
return "", err
Expand All @@ -275,29 +315,30 @@ func (p *KeyBrokerService) GetKbsPodIP(ctx context.Context, cfg *envconf.Config)

resources := client.Resources(namespace)

podList := &corev1.PodList{}
err = resources.List(context.TODO(), podList)
// Get the service associated with the deployment
serviceList := &corev1.ServiceList{}
err = resources.List(context.TODO(), serviceList)
if err != nil {
err = fmt.Errorf("Error listing pods: %w\n", err)
err = fmt.Errorf("Error listing services: %w\n", err)
log.Errorf("%v", err)
return "", err
}

var matchingPod *corev1.Pod
for i := range podList.Items {
pod := &podList.Items[i]
if pod.Labels["app"] == deploymentName {
matchingPod = pod
var matchingService *corev1.Service
for i := range serviceList.Items {
service := &serviceList.Items[i]
if service.Name == deploymentName {
matchingService = service
break
}
}

if matchingPod == nil {
return "", fmt.Errorf("No pod with label selector found")
if matchingService == nil {
return "", fmt.Errorf("No service with label selector found")
}

fmt.Printf("Pod IP: %s\n", matchingPod.Status.PodIP)
return matchingPod.Status.PodIP, nil
fmt.Printf("KBS Service IP: %s\n", matchingService.Spec.ClusterIP)
return matchingService.Spec.ClusterIP, nil
}

func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, props map[string]string) error {
Expand All @@ -313,19 +354,30 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop
return err
}

newDirectory := "kbs/config/kubernetes/overlays"
newDirectory := "kbs/kbs/config/kubernetes"
err = os.Chdir(newDirectory)
if err != nil {
err = fmt.Errorf("Error changing the working directory: %w\n", err)
log.Errorf("%v", err)
return err
}

log.Info("Install Kbs")
if err := p.installOverlay.Apply(ctx, cfg); err != nil {
// Replace this to use install overlay
cmd := exec.Command("kubectl", "apply", "-k", "overlays")
cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
stdoutStderr, err := cmd.CombinedOutput()
log.Tracef("%v, output: %s", cmd, stdoutStderr)
if err != nil {
return err
}

/*
log.Info("Install Kbs")
if err := p.installOverlay.Apply(ctx, cfg); err != nil {
return err
}
*/

// Return to the original working directory.
err = os.Chdir(originalDir)
if err != nil {
Expand All @@ -347,15 +399,15 @@ func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) erro
}

// Remove kbs deployment
newDirectory := "kbs/config/kubernetes/overlays"
newDirectory := "kbs/kbs/config/kubernetes"
err = os.Chdir(newDirectory)
if err != nil {
err = fmt.Errorf("Error changing the working directory: %w\n", err)
log.Errorf("%v", err)
return err
}

log.Info("Install Kbs")
log.Info("Delete Kbs deployment")
if err := p.installOverlay.Delete(ctx, cfg); err != nil {
return err
}
Expand Down Expand Up @@ -582,7 +634,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string)
for _, o := range metaList {
obj, _ := o.(k8s.Object)
fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName())
if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*6)); err != nil {
if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil {
return err
}
fmt.Printf("pod '%s' is Ready\n", obj.GetName())
Expand Down

0 comments on commit 2005f51

Please sign in to comment.