From 8e06c35fcc57d69f5e307897477967373c008395 Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Thu, 1 Feb 2024 13:54:25 +0530 Subject: [PATCH] Provisioner: Add kbs provisioning steps in CI Fixes: #1676 Signed-off-by: Kartik Joshi --- .github/workflows/azure-e2e-test.yml | 27 +++++++++++++++++++ .../test/provisioner/provision.go | 17 +++++++----- src/cloud-api-adaptor/versions.yaml | 5 +++- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/azure-e2e-test.yml b/.github/workflows/azure-e2e-test.yml index 3c1af3ea3e..4dc03dc0d9 100644 --- a/.github/workflows/azure-e2e-test.yml +++ b/.github/workflows/azure-e2e-test.yml @@ -25,6 +25,12 @@ on: caa-image: type: string description: prebuilt caa image + kbs-image-id: + type: string + description: prebuild kbs image + kbs-image-tag: + type: string + description: prebuild kbs image tag jobs: generate-podvm-image-version: @@ -116,10 +122,17 @@ jobs: with: go-version: ${{ env.GO_VERSION }} + - name: Extract provisioner environment + run: | + echo "KBS_IMAGE=$(yq -e '.oci.kbs.registry' versions.yaml)" >> "$GITHUB_ENV" + echo "KBS_IMAGE_TAG=$(yq -e '.oci.kbs.tag' versions.yaml)" >> "$GITHUB_ENV" + - name: Create provisioner file env: AZURE_IMAGE_ID: ${{ github.event.inputs.podvm-image-id || format('/CommunityGalleries/{0}/images/{1}/Versions/{2}', vars.AZURE_COMMUNITY_GALLERY_NAME, vars.AZURE_PODVM_IMAGE_DEF_NAME, needs.generate-podvm-image-version.outputs.image-version) }} CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}" + KBS_IMAGE: ${{ github.event.inputs.kbs-image-id || env.KBS_IMAGE }} + KBS_IMAGE_TAG: ${{ github.event.inputs.kbs-image-tag || env.KBS_IMAGE_TAG}} run: | cat << EOF > "$TEST_PROVISION_FILE" AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" @@ -132,6 +145,8 @@ jobs: IS_CI_MANAGED_CLUSTER="true" MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}" CAA_IMAGE="${CAA_IMAGE}" + KBS_IMAGE="${KBS_IMAGE}" + KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" EOF cat "$TEST_PROVISION_FILE" # assert that no variable is unset @@ -148,6 +163,17 @@ jobs: ${{ env.TEST_PROVISION_FILE }} name: e2e-configuration + - name: Extract kbs reference + run: echo "KBS_VERSION=$(yq -e '.git.kbs.reference' versions.yaml)" >> "$GITHUB_ENV" + + - name: Checkout kbs Repository + run: | + git clone https://github.com/confidential-containers/trustee test/trustee + pushd test/trustee + git checkout "${KBS_VERSION}" + popd + + - uses: azure/login@v1 name: 'Az CLI login' with: @@ -206,6 +232,7 @@ jobs: - name: Run e2e test env: TEST_PROVISION: "no" + DEPLOY_KBS: "yes" run: | # Since we install the cluster in parallel, we need to get the credentials here. az aks get-credentials \ diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 0c8d57ff43..4be6baefb8 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -80,6 +80,9 @@ type InstallOverlay interface { // Waiting timeout for bringing up the pod const PodWaitTimeout = time.Second * 30 +// trustee repo related base path +const TRUSTEE_REPO_PATH = "../trustee" + func saveToFile(filename string, content []byte) error { // Save contents to file err := os.WriteFile(filename, content, 0644) @@ -94,7 +97,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { // Create secret content := []byte("This is my cluster name: " + clusterName) - filePath := "trustee/kbs/config/kubernetes/overlays/key.bin" + filePath := filepath.Join(TRUSTEE_REPO_PATH, "/kbs/config/kubernetes/overlays/key.bin") // Create the file. file, err := os.Create(filePath) if err != nil { @@ -120,9 +123,9 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } fmt.Println(k8sCnfDir) - kbsCert := filepath.Join(k8sCnfDir, "trustee/kbs/config/kubernetes/base/kbs.pem") + kbsCert := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.pem") if _, err := os.Stat(kbsCert); os.IsNotExist(err) { - kbsKey := filepath.Join(k8sCnfDir, "trustee/kbs/config/kubernetes/base/kbs.key") + kbsKey := filepath.Join(k8sCnfDir, TRUSTEE_REPO_PATH, "kbs/config/kubernetes/base/kbs.key") keyOutputFile, err := os.Create(kbsKey) if err != nil { err = fmt.Errorf("creating key file: %w\n", err) @@ -174,7 +177,7 @@ func NewKeyBrokerService(clusterName string) (*KeyBrokerService, error) { } - overlay, err := NewBaseKbsInstallOverlay("trustee") + overlay, err := NewBaseKbsInstallOverlay(TRUSTEE_REPO_PATH) if err != nil { return nil, err } @@ -339,7 +342,7 @@ func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Conf return "", fmt.Errorf("Service %s is not of type NodePort", "kbs") } - var nodePort + var nodePort int32 // Extract NodePort if len(service.Spec.Ports) > 0 { nodePort = service.Spec.Ports[0].NodePort @@ -367,7 +370,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop } // Create kustomize pointer for overlay directory with updated changes - tmpoverlay, err := NewKbsInstallOverlay("trustee") + tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH) if err != nil { return err } @@ -381,7 +384,7 @@ func (p *KeyBrokerService) Deploy(ctx context.Context, cfg *envconf.Config, prop func (p *KeyBrokerService) Delete(ctx context.Context, cfg *envconf.Config) error { // Create kustomize pointer for overlay directory with updated changes - tmpoverlay, err := NewKbsInstallOverlay("trustee") + tmpoverlay, err := NewKbsInstallOverlay(TRUSTEE_REPO_PATH) if err != nil { return err } diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml index d5de5432ad..91e60a0f89 100644 --- a/src/cloud-api-adaptor/versions.yaml +++ b/src/cloud-api-adaptor/versions.yaml @@ -41,10 +41,13 @@ git: opa: url: https://github.com/open-policy-agent/opa reference: v0.58.0 + kbs: + url: https://github.com/confidential-containers/trustee + reference: dc01f454264fb4350e5f69eba05683a9a1882c41 oci: pause: registry: docker://registry.k8s.io/pause tag: 3.6 kbs: registry: ghcr.io/confidential-containers/key-broker-service - tag: v0.8.2 + tag: dc01f454264fb4350e5f69eba05683a9a1882c41