Update ci teardown to cluster init (#150)

1. Add new action build-cluster-init and run-cluster-init to be used by both turing and cluster-init ci.
2. Add new e2e test in cluster-init ci to smoke test knative service.
3. Shifted tear down test from turing to cluster-init

Co-authored-by: leonlnj <ningjie.lee@gojek.com>

.github/actions/build-cluster-init/action.yaml

@@ -0,0 +1,38 @@
+name: Build Cluster Init
+description: Set up Turing Cluster Init
+
+inputs:
+  artifact_retention_days: 
+    required: false
+    description: 'Artifact retention days'
+    default: 7
+outputs:
+  cluster-init-version:
+    description: 'Cluster Init Version built'
+    value: ${{ steps.build-image.outputs.cluster-init-version }}
+
+runs:
+  using: composite
+  steps:
+      - name: Build Docker image
+        id: build-image
+        working-directory: infra/cluster-init
+        shell: bash
+        run: |
+          set -o pipefail
+          make build-image | tee output.log
+          echo "::set-output name=cluster-init-version::$(sed -n 's%turing-cluster-init version: \(.*\)%\1%p' output.log)"
+
+      - name: Save Docker image
+        shell: bash
+        run: |
+          docker image save \
+            --output cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar \
+            cluster-init:${{ steps.build-image.outputs.cluster-init-version }}
+
+      - name: Publish Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar
+          path: cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar
+          retention-days: ${{ inputs.artifact_retention_days }}

.github/actions/run-cluster-init/action.yaml

@@ -0,0 +1,79 @@
+name: Run Cluster Init
+description: Set up Turing Cluster Init
+
+inputs:
+  cluster_name: 
+    required: true
+    description: 'Name of Cluster'
+    default: ''
+  istio_version:
+    required: true
+    description: 'Istio Version'
+    default: ''
+  knative_version:
+    required: true
+    description: 'Knative Version'
+    default: ''
+  knative_istio_version:
+    required: true
+    description: 'Knative Istio Version'
+    default: ''
+  local_registry:
+    required: true
+    description: 'Endpoint of local registry'
+    default: ''
+  cluster_init_version:
+    required: true
+    description: 'Version of cluster to install, tar file has to follow naming - cluster-init.(CLUSTER_INIT_VERSION).tar '
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: "Setup local k8s cluster"
+      uses: AbsaOSS/k3d-action@v1.5.0
+      with:
+        cluster-name: ${{ inputs.cluster_name }}
+        use-default-registry: true
+        args: >-
+          --servers 1
+          --agents 3
+          --port 80:80@loadbalancer
+          --k3s-server-arg "--no-deploy=traefik,metrics-server"
+
+    - name: Publish images to local registry
+      env:
+        DOCKER_REPOSITORY: ${{ inputs.local_registry }}/${{ github.repository }}
+        CLUSTER_INIT_VERSION: ${{ inputs.cluster_init_version }}
+      shell: bash
+      run: |
+        # Cluster init
+        docker image load --input cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar
+        docker tag \
+          cluster-init:${{ env.CLUSTER_INIT_VERSION }} \
+          ${{ env.DOCKER_REPOSITORY }}/cluster-init:${{ env.CLUSTER_INIT_VERSION }}
+        docker push ${{ env.DOCKER_REPOSITORY }}/cluster-init:${{ env.CLUSTER_INIT_VERSION }}
+
+    - name: Install Infrastructure
+      env:
+        CLUSTER_INIT_VERSION: ${{ inputs.cluster_init_version }}
+      shell: bash
+      run: |
+        kubectl create ns infrastructure
+        helm upgrade turing-init infra/charts/turing-init \
+          --namespace infrastructure \
+          --set image.registry=${{ inputs.local_registry }}/ \
+          --set image.repository=${{ github.repository }}/cluster-init \
+          --set image.tag=${{ env.CLUSTER_INIT_VERSION }} \
+          --set knative.domains="127.0.0.1.nip.io" \
+          --set knative.registriesSkippingTagResolving=${{ inputs.local_registry }} \
+          --install \
+          --wait
+
+        # wait for install infra job to finish
+        kubectl logs -n infrastructure -f $(kubectl get pod --namespace infrastructure | grep -v 'NAME' | grep -v 'spark' | head -n 1 | awk '{print $1}')
+        kubectl get pod --all-namespaces
+        kubectl get svc --all-namespaces
+        kubectl wait -n infrastructure --for=condition=complete --timeout=10m job/turing-init-spark-operator-webhook-init
+        # Might fail the first time but the 2nd run should work, rarely fails on the first try though.
+        kubectl wait -n infrastructure --for=condition=complete --timeout=10m job/turing-init-init

.github/workflows/cluster-init.yaml

@@ -35,43 +35,117 @@ jobs:
         with:
           prefix: cluster-init/
 
-  publish:
-    # Automatically publish release and pre-release artifacts.
-    #
-    # As for dev releases, make it possible to publish artifacts
-    # manually by approving 'deployment' in the 'manual' environment.
-    #
-    # Dev build can be released either from the 'main' branch or
-    # by running this workflow manually with `workflow_dispatch` event.
-    if: >-
-      contains('release,pre-release', needs.release-rules.outputs.release-type)
-        || ( github.event_name != 'pull_request' )
-        || ( github.event.pull_request.head.repo.full_name == github.repository )
-    environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }}
+  build-cluster-init:
     runs-on: ubuntu-latest
-    needs:
-      - release-rules
+    outputs:
+      cluster-init-version: ${{ steps.build-cluster-init.outputs.cluster-init-version }}
     steps:
-      - uses: actions/checkout@v2
+      - name: Check out code
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
+      - name: Run action build-cluster-init
+        id: build-cluster-init
+        uses: ./.github/actions/build-cluster-init
+
+  test-e2e:
+    runs-on: ubuntu-latest
+    env:
+      CLUSTER_INIT_VERSION: ${{ needs.build-cluster-init.outputs.cluster-init-version }}
+      CLUSTER_NAME: turing-e2e
+      ISTIO_VERSION: 1.9.9
+      KNATIVE_VERSION: v0.18.3
+      KNATIVE_ISTIO_VERSION: v0.18.1
+      LOCAL_REGISTRY: registry.localhost:5000
+    needs:
+      - build-cluster-init
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
 
-      - name: Log in to the Container registry
-        uses: docker/login-action@v1
+      - name: Download Cluster Init Docker tar archieve
+        uses: actions/download-artifact@v2
+        with:
+          name: cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar
+
+      - name: Run action cluster-init
+        uses: ./.github/actions/run-cluster-init
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          cluster_name: ${{ env.CLUSTER_NAME}}
+          istio_version: ${{ env.ISTIO_VERSION}}
+          knative_version: ${{ env.KNATIVE_VERSION}}
+          knative_istio_version: ${{ env.KNATIVE_ISTIO_VERSION}}
+          local_registry: ${{ env.LOCAL_REGISTRY}}
+          cluster_init_version: ${{ env.CLUSTER_INIT_VERSION }}
 
-      - name: Build Docker Image
-        id: build
-        working-directory: infra/cluster-init
-        env:
-          DOCKER_REGISTRY: ghcr.io/${{ github.repository }}
+      - name: Smoke Test
         run: |
-          set -o pipefail
-          make build-image | tee output.log
-          echo "::set-output name=cluster-init-image::$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)"
+          # Create hello world knative service
+          tee service.yaml <<EOF 
+          apiVersion: serving.knative.dev/v1
+          kind: Service
+          metadata:
+            name: helloworld-go 
+            namespace: default
+          spec:
+            template:
+              spec:
+                containers:
+                  - image: gcr.io/knative-samples/helloworld-go
+                    env:
+                      - name: TARGET
+                        value: "Hello Knative Serving is up and running!!"
+          EOF
+
+          kubectl apply -f service.yaml
+
+          # wait till service is up
+          timeout --foreground 120 bash -c 'until kubectl get service.serving.knative.dev/helloworld-go --output=jsonpath='{.status.conditions[1]}'  | grep "True"; do : ; done'
+          kubectl get ksvc
+
+          curl -f http://helloworld-go.default.127.0.0.1.nip.io/
+          kubectl delete service helloworld-go 
+      
+      # Invoke helm hooks on delete
+      - name: Tear down infrastructure job
+        run: helm delete --namespace infrastructure turing-init --timeout 15m
+
+  publish:
+      # Automatically publish release and pre-release artifacts.
+      #
+      # As for dev releases, make it possible to publish artifacts
+      # manually by approving 'deployment' in the 'manual' environment.
+      #
+      # Dev build can be released either from the 'main' branch or
+      # by running this workflow manually with `workflow_dispatch` event.
+      if: >-
+        contains('release,pre-release', needs.release-rules.outputs.release-type)
+          || ( github.event_name != 'pull_request' )
+          || ( github.event.pull_request.head.repo.full_name == github.repository )
+      environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }}
+      runs-on: ubuntu-latest
+      needs:
+        - release-rules
+        - build-cluster-init
+        - test-e2e
+      steps:
+        - uses: actions/checkout@v2
+          with:
+            fetch-depth: 0
+
+        - name: Log in to the Container registry
+          uses: docker/login-action@v1
+          with:
+            registry: ghcr.io
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Download Cluster Init Docker tar archieve
+          uses: actions/download-artifact@v2
+          with:
+            name: cluster-init.${{ needs.build-cluster-init.outputs.cluster-init-version }}.tar
+
+        - name: Publish Cluster Init Docker Image
+          run: docker push ${{ steps.build.outputs.cluster-init-image }}
 
-      - name: Publish Cluster Init Docker Image
-        run: docker push ${{ steps.build.outputs.cluster-init-image }}