diff --git a/install/overlays/azure/kustomization.yaml b/install/overlays/azure/kustomization.yaml index 503c9ea31d..f9a6a09474 100644 --- a/install/overlays/azure/kustomization.yaml +++ b/install/overlays/azure/kustomization.yaml @@ -32,13 +32,12 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name - - AA_KBC_PARAMS="" #set + - AA_KBC_PARAMS="" #set KBC params for podvm #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 #- AZURE_INSTANCE_SIZES="" # comma separated #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm - #- AA_KBC_PARAMS="" # Uncomment and set if you want to set KBC params for podvm #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150 ##TLS_SETTINGS #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 5dc326c228..23c3ec91e0 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -116,7 +116,12 @@ func TestMain(m *testing.M) { testEnv.Setup(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { log.Info("Do setup") var err error + // Get properties + props := provisioner.GetProperties(ctx, cfg) + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + return ctx, fmt.Errorf("kbs image not provided") + } if shouldProvisionCluster { log.Info("Cluster provisioning") @@ -129,14 +134,9 @@ func TestMain(m *testing.M) { } } - props := provisioner.GetProperties(ctx, cfg) var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") - if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { - return ctx, fmt.Errorf("kbs image not provided") - } - if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"]); err != nil { return ctx, err } @@ -144,12 +144,12 @@ func TestMain(m *testing.M) { if err = keyBrokerService.Deploy(ctx, cfg, props); err != nil { return ctx, err } - var kbsPodIP string - if kbsPodIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { + var kbsSvcIP string + if kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(ctx, cfg); err != nil { return ctx, err } - kbsparams = "cc_kbc::http:" + kbsPodIP + ":8080" + kbsparams = "cc_kbc::http:" + kbsSvcIP + ":8080" log.Infof("KBS PARAMS%s:", kbsparams) } diff --git a/test/provisioner/azure/provision_azure.properties b/test/provisioner/azure/provision_azure.properties index 0b19cff4c5..5e8bf51e01 100644 --- a/test/provisioner/azure/provision_azure.properties +++ b/test/provisioner/azure/provision_azure.properties @@ -13,5 +13,5 @@ SSH_USERNAME="" AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" -KBS_IMAGE="ghcr.io/confidential-containers/staged-images/kbs" -KBS_IMAGE_TAG="latest" +KBS_IMAGE="${KBS_IMAGE}" +KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" diff --git a/test/provisioner/provision.go b/test/provisioner/provision.go index 5420b6fd96..1a5c9ce839 100644 --- a/test/provisioner/provision.go +++ b/test/provisioner/provision.go @@ -10,7 +10,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -78,40 +77,19 @@ type InstallOverlay interface { Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error } -func runCommand(command string, stdout io.Writer, stderr io.Writer, args ...string) error { - cmd := exec.Command(command, args...) - cmd.Stdout = stdout - cmd.Stderr = stderr - - fmt.Printf("Running command: %s %v\n", command, args) - - if err := cmd.Run(); err != nil { - err = fmt.Errorf(fmt.Sprintf("Error running command: %s %v - %s", command, args, err)) - - log.Errorf("%v", err) - return err - } - - return nil -} +// Waiting timeout for bringing up the pod +const PodWaitTimeout = time.Second * 30 func saveToFile(filename string, content []byte) error { // Save contents to file err := os.WriteFile(filename, content, 0644) if err != nil { - return fmt.Errorf("error writing contents to file: %w", err) + return fmt.Errorf("writing contents to file: %w", err) } return nil } func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { - // Clone kbs repo - repoURL := "https://github.com/confidential-containers/kbs" - - if err := runCommand("git", os.Stdout, os.Stderr, "clone", repoURL); err != nil { - return nil, err - } - log.Info("creating key.bin") // Create secret @@ -120,7 +98,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Create the file. file, err := os.Create(filePath) if err != nil { - err = fmt.Errorf("Error creating file: %w\n", err) + err = fmt.Errorf("creating file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -129,14 +107,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Write the content to the file. err = saveToFile(filePath, content) if err != nil { - err = fmt.Errorf("Error writing to the file: %w\n", err) + err = fmt.Errorf("writing to the file: %w\n", err) log.Errorf("%v", err) return nil, err } k8sCnfDir, err := os.Getwd() if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) + err = fmt.Errorf("getting the current working directory: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -147,7 +125,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { kbsKey := filepath.Join(k8sCnfDir, "kbs/kbs/config/kubernetes/base/kbs.key") keyOutputFile, err := os.Create(kbsKey) if err != nil { - err = fmt.Errorf("Error creating key file: %w\n", err) + err = fmt.Errorf("creating key file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -155,7 +133,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { _, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { - err = fmt.Errorf("Error generating Ed25519 key pair: %w\n", err) + err = fmt.Errorf("generating Ed25519 key pair: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -168,7 +146,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Save private key to file err = saveToFile(kbsKey, privateKeyPEM) if err != nil { - err = fmt.Errorf("Error saving private key to file: %w\n", err) + err = fmt.Errorf("saving private key to file: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -176,7 +154,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { publicKey := privateKey.Public().(ed25519.PublicKey) publicKeyX509, err := x509.MarshalPKIXPublicKey(publicKey) if err != nil { - err = fmt.Errorf("Error generating Ed25519 public key: %w\n", err) + err = fmt.Errorf("generating Ed25519 public key: %w\n", err) log.Errorf("%v", err) return nil, err } @@ -189,14 +167,14 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Save public key to file err = saveToFile(kbsCert, publicKeyPEM) if err != nil { - err = fmt.Errorf("Error saving public key to file: %w\n", err) + err = fmt.Errorf("saving public key to file: %w\n", err) log.Errorf("%v", err) return nil, err } } - overlay, err := NewKbsInstallOverlay("kbs") + overlay, err := NewBaseKbsInstallOverlay("kbs") if err != nil { return nil, err } @@ -262,7 +240,7 @@ func GetInstallOverlay(provider string, installDir string) (InstallOverlay, erro return overlayFunc(installDir, provider) } -func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { +func NewBaseKbsInstallOverlay(installDir string) (InstallOverlay, error) { log.Info("Creating kbs install overlay") overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/base")) if err != nil { @@ -274,6 +252,18 @@ func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { }, nil } +func NewKbsInstallOverlay(installDir string) (InstallOverlay, error) { + log.Info("Creating kbs install overlay") + overlay, err := NewKustomizeOverlay(filepath.Join(installDir, "kbs/config/kubernetes/overlays")) + if err != nil { + return nil, err + } + + return &KbsInstallOverlay{ + overlay: overlay, + }, nil +} + func (lio *KbsInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { return lio.overlay.Apply(ctx, cfg) } @@ -289,7 +279,7 @@ func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, pro return err } - log.Infof("Updating CAA image tag with %q", props["KBS_IMAGE_TAG"]) + log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"]) if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { return err } @@ -319,7 +309,7 @@ func (p *KeyBrokerService) GetKbsSvcIP(ctx context.Context, cfg *envconf.Config) serviceList := &corev1.ServiceList{} err = resources.List(context.TODO(), serviceList) if err != nil { - err = fmt.Errorf("Error listing services: %w\n", err) + err = fmt.Errorf("listing services: %w\n", err) log.Errorf("%v", err) return "", err } @@ -347,42 +337,14 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop return err } - originalDir, err := os.Getwd() + // Create kustomize pointer for overlay directory with updated changes + tmpoverlay, err := NewKbsInstallOverlay("kbs") if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) - log.Errorf("%v", err) return err } - newDirectory := "kbs/kbs/config/kubernetes" - err = os.Chdir(newDirectory) - if err != nil { - err = fmt.Errorf("Error changing the working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - // Replace this to use install overlay - cmd := exec.Command("kubectl", "apply", "-k", "overlays") - cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) - stdoutStderr, err := cmd.CombinedOutput() - log.Tracef("%v, output: %s", cmd, stdoutStderr) - if err != nil { - return err - } - - /* - log.Info("Install Kbs") - if err := p.installOverlay.Apply(ctx, cfg); err != nil { - return err - } - */ - - // Return to the original working directory. - err = os.Chdir(originalDir) - if err != nil { - err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) - log.Errorf("%v", err) + log.Info("Install Kbs") + if err := tmpoverlay.Apply(ctx, cfg); err != nil { return err } @@ -390,46 +352,16 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop } func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error { - - originalDir, err := os.Getwd() + // Create kustomize pointer for overlay directory with updated changes + tmpoverlay, err := NewKbsInstallOverlay("kbs") if err != nil { - err = fmt.Errorf("Error getting the current working directory: %w\n", err) - log.Errorf("%v", err) return err } - // Remove kbs deployment - newDirectory := "kbs/kbs/config/kubernetes" - err = os.Chdir(newDirectory) - if err != nil { - err = fmt.Errorf("Error changing the working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - log.Info("Delete Kbs deployment") - if err := p.installOverlay.Delete(ctx, cfg); err != nil { - return err - } - - // Return to the original working directory. - err = os.Chdir(originalDir) - if err != nil { - err = fmt.Errorf("Error changing back to the original working directory: %w\n", err) - log.Errorf("%v", err) - return err - } - - // remove kbs repo - directoryPath := "kbs" - - err = os.RemoveAll(directoryPath) - if err != nil { - err = fmt.Errorf("Error deleting directory: %w\n", err) - log.Errorf("%v", err) + log.Info("Uninstall the cloud-api-adaptor") + if err = tmpoverlay.Delete(ctx, cfg); err != nil { return err } - return nil } @@ -634,7 +566,7 @@ func AllPodsRunning(ctx context.Context, cfg *envconf.Config, namespace string) for _, o := range metaList { obj, _ := o.(k8s.Object) fmt.Printf("Wait pod '%s' status for Ready\n", obj.GetName()) - if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(time.Second*15)); err != nil { + if err := wait.For(conditions.New(resources).PodReady(obj), wait.WithTimeout(PodWaitTimeout)); err != nil { return err } fmt.Printf("pod '%s' is Ready\n", obj.GetName()) diff --git a/test/tools/provisioner-cli/main.go b/test/tools/provisioner-cli/main.go index 3e3e85421f..c1073b80cd 100644 --- a/test/tools/provisioner-cli/main.go +++ b/test/tools/provisioner-cli/main.go @@ -63,6 +63,11 @@ func main() { } if *action == "provision" { + props := provisioner.GetProperties(context.TODO(), cfg) + if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { + log.Fatal("kbs image not provided") + } + log.Info("Creating VPC...") if err := provisioner.CreateVPC(context.TODO(), cfg); err != nil { log.Fatal(err) @@ -83,11 +88,33 @@ func main() { } } + log.Info("Deploying kbs") + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + if err != nil { + log.Fatal(err) + } + + if err = keyBrokerService.Deploy(context.TODO(), cfg, props); err != nil { + log.Fatal(err) + } + + var kbsSvcIP string + kbsSvcIP, err = keyBrokerService.GetKbsSvcIP(context.TODO(), cfg) + if err != nil { + log.Fatal(err) + } + + kbsparams := "cc_kbc::http:" + kbsSvcIP + ":8080" + log.Infof("KBS PARAMS: %s", kbsparams) + + props = provisioner.GetProperties(context.TODO(), cfg) + props["AA_KBC_PARAMS"] = kbsparams + cloudAPIAdaptor, err := pv.NewCloudAPIAdaptor(cloudProvider, installDirectory) if err != nil { log.Fatal(err) } - if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)); err != nil { + if err := cloudAPIAdaptor.Deploy(context.TODO(), cfg, props); err != nil { log.Fatal(err) } } @@ -128,7 +155,7 @@ func main() { if kubeconfigPath == "" { log.Fatal("Unabled to find a kubeconfig file") } - cfg := envconf.NewWithKubeConfig(kubeconfigPath) + cfg = envconf.NewWithKubeConfig(kubeconfigPath) err = deployer.Deploy(context.TODO(), cfg, provisioner.GetProperties(context.TODO(), cfg)) if err != nil { @@ -143,6 +170,16 @@ func main() { log.Fatal(err) } + props := provisioner.GetProperties(context.TODO(), cfg) + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"]) + if err != nil { + log.Fatal(err) + } + + if err = keyBrokerService.Delete(context.TODO(), cfg); err != nil { + log.Fatal(err) + } + kubeconfigPath := kconf.ResolveKubeConfigFile() if kubeconfigPath == "" { log.Fatal("Unabled to find a kubeconfig file")