diff --git a/.github/actions/build-cluster-init/action.yaml b/.github/actions/build-cluster-init/action.yaml new file mode 100644 index 000000000..81a40f2a1 --- /dev/null +++ b/.github/actions/build-cluster-init/action.yaml @@ -0,0 +1,38 @@ +name: Build Cluster Init +description: Set up Turing Cluster Init + +inputs: + artifact_retention_days: + required: false + description: 'Artifact retention days' + default: 7 +outputs: + cluster-init-version: + description: 'Cluster Init Version built' + value: ${{ steps.build-image.outputs.cluster-init-version }} + +runs: + using: composite + steps: + - name: Build Docker image + id: build-image + working-directory: infra/cluster-init + shell: bash + run: | + set -o pipefail + make build-image | tee output.log + echo "::set-output name=cluster-init-version::$(sed -n 's%turing-cluster-init version: \(.*\)%\1%p' output.log)" + + - name: Save Docker image + shell: bash + run: | + docker image save \ + --output cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar \ + cluster-init:${{ steps.build-image.outputs.cluster-init-version }} + + - name: Publish Artifact + uses: actions/upload-artifact@v2 + with: + name: cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar + path: cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar + retention-days: ${{ inputs.artifact_retention_days }} diff --git a/.github/actions/run-cluster-init/action.yaml b/.github/actions/run-cluster-init/action.yaml new file mode 100644 index 000000000..107d60c79 --- /dev/null +++ b/.github/actions/run-cluster-init/action.yaml @@ -0,0 +1,79 @@ +name: Run Cluster Init +description: Set up Turing Cluster Init + +inputs: + cluster_name: + required: true + description: 'Name of Cluster' + default: '' + istio_version: + required: true + description: 'Istio Version' + default: '' + knative_version: + required: true + description: 'Knative Version' + default: '' + knative_istio_version: + required: true + description: 'Knative Istio Version' + default: '' + local_registry: + required: true + description: 'Endpoint of local registry' + default: '' + cluster_init_version: + required: true + description: 'Version of cluster to install, tar file has to follow naming - cluster-init.(CLUSTER_INIT_VERSION).tar ' + default: '' + +runs: + using: composite + steps: + - name: "Setup local k8s cluster" + uses: AbsaOSS/k3d-action@v1.5.0 + with: + cluster-name: ${{ inputs.cluster_name }} + use-default-registry: true + args: >- + --servers 1 + --agents 3 + --port 80:80@loadbalancer + --k3s-server-arg "--no-deploy=traefik,metrics-server" + + - name: Publish images to local registry + env: + DOCKER_REPOSITORY: ${{ inputs.local_registry }}/${{ github.repository }} + CLUSTER_INIT_VERSION: ${{ inputs.cluster_init_version }} + shell: bash + run: | + # Cluster init + docker image load --input cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar + docker tag \ + cluster-init:${{ env.CLUSTER_INIT_VERSION }} \ + ${{ env.DOCKER_REPOSITORY }}/cluster-init:${{ env.CLUSTER_INIT_VERSION }} + docker push ${{ env.DOCKER_REPOSITORY }}/cluster-init:${{ env.CLUSTER_INIT_VERSION }} + + - name: Install Infrastructure + env: + CLUSTER_INIT_VERSION: ${{ inputs.cluster_init_version }} + shell: bash + run: | + kubectl create ns infrastructure + helm upgrade turing-init infra/charts/turing-init \ + --namespace infrastructure \ + --set image.registry=${{ inputs.local_registry }}/ \ + --set image.repository=${{ github.repository }}/cluster-init \ + --set image.tag=${{ env.CLUSTER_INIT_VERSION }} \ + --set knative.domains="127.0.0.1.nip.io" \ + --set knative.registriesSkippingTagResolving=${{ inputs.local_registry }} \ + --install \ + --wait + + # wait for install infra job to finish + kubectl logs -n infrastructure -f $(kubectl get pod --namespace infrastructure | grep -v 'NAME' | grep -v 'spark' | head -n 1 | awk '{print $1}') + kubectl get pod --all-namespaces + kubectl get svc --all-namespaces + kubectl wait -n infrastructure --for=condition=complete --timeout=10m job/turing-init-spark-operator-webhook-init + # Might fail the first time but the 2nd run should work, rarely fails on the first try though. + kubectl wait -n infrastructure --for=condition=complete --timeout=10m job/turing-init-init diff --git a/.github/workflows/cluster-init.yaml b/.github/workflows/cluster-init.yaml index e4eada842..9672d44a2 100644 --- a/.github/workflows/cluster-init.yaml +++ b/.github/workflows/cluster-init.yaml @@ -35,43 +35,117 @@ jobs: with: prefix: cluster-init/ - publish: - # Automatically publish release and pre-release artifacts. - # - # As for dev releases, make it possible to publish artifacts - # manually by approving 'deployment' in the 'manual' environment. - # - # Dev build can be released either from the 'main' branch or - # by running this workflow manually with `workflow_dispatch` event. - if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) - || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) - environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} + build-cluster-init: runs-on: ubuntu-latest - needs: - - release-rules + outputs: + cluster-init-version: ${{ steps.build-cluster-init.outputs.cluster-init-version }} steps: - - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v2 with: fetch-depth: 0 + + - name: Run action build-cluster-init + id: build-cluster-init + uses: ./.github/actions/build-cluster-init + + test-e2e: + runs-on: ubuntu-latest + env: + CLUSTER_INIT_VERSION: ${{ needs.build-cluster-init.outputs.cluster-init-version }} + CLUSTER_NAME: turing-e2e + ISTIO_VERSION: 1.9.9 + KNATIVE_VERSION: v0.18.3 + KNATIVE_ISTIO_VERSION: v0.18.1 + LOCAL_REGISTRY: registry.localhost:5000 + needs: + - build-cluster-init + steps: + - name: Check out code + uses: actions/checkout@v2 - - name: Log in to the Container registry - uses: docker/login-action@v1 + - name: Download Cluster Init Docker tar archieve + uses: actions/download-artifact@v2 + with: + name: cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar + + - name: Run action cluster-init + uses: ./.github/actions/run-cluster-init with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + cluster_name: ${{ env.CLUSTER_NAME}} + istio_version: ${{ env.ISTIO_VERSION}} + knative_version: ${{ env.KNATIVE_VERSION}} + knative_istio_version: ${{ env.KNATIVE_ISTIO_VERSION}} + local_registry: ${{ env.LOCAL_REGISTRY}} + cluster_init_version: ${{ env.CLUSTER_INIT_VERSION }} - - name: Build Docker Image - id: build - working-directory: infra/cluster-init - env: - DOCKER_REGISTRY: ghcr.io/${{ github.repository }} + - name: Smoke Test run: | - set -o pipefail - make build-image | tee output.log - echo "::set-output name=cluster-init-image::$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" + # Create hello world knative service + tee service.yaml <- + contains('release,pre-release', needs.release-rules.outputs.release-type) + || ( github.event_name != 'pull_request' ) + || ( github.event.pull_request.head.repo.full_name == github.repository ) + environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} + runs-on: ubuntu-latest + needs: + - release-rules + - build-cluster-init + - test-e2e + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Log in to the Container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Download Cluster Init Docker tar archieve + uses: actions/download-artifact@v2 + with: + name: cluster-init.${{ needs.build-cluster-init.outputs.cluster-init-version }}.tar + + - name: Publish Cluster Init Docker Image + run: docker push ${{ steps.build.outputs.cluster-init-image }} - - name: Publish Cluster Init Docker Image - run: docker push ${{ steps.build.outputs.cluster-init-image }} diff --git a/.github/workflows/turing.yaml b/.github/workflows/turing.yaml index ae36c5bdd..7e24ff789 100644 --- a/.github/workflows/turing.yaml +++ b/.github/workflows/turing.yaml @@ -305,33 +305,19 @@ jobs: build-cluster-init: runs-on: ubuntu-latest outputs: - cluster-init-version: ${{ steps.build-image.outputs.cluster-init-version }} + cluster-init-version: ${{ steps.build-cluster-init.outputs.cluster-init-version }} steps: - name: Check out code uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Build Docker image - id: build-image - working-directory: infra/cluster-init - run: | - set -o pipefail - make build-image | tee output.log - echo "::set-output name=cluster-init-version::$(sed -n 's%turing-cluster-init version: \(.*\)%\1%p' output.log)" - - - name: Save Docker image - run: | - docker image save \ - --output cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar \ - cluster-init:${{ steps.build-image.outputs.cluster-init-version }} - - - name: Publish Artifact - uses: actions/upload-artifact@v2 + - name: Run action build-cluster-init + id: build-cluster-init + uses: ./.github/actions/build-cluster-init with: - name: cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar - path: cluster-init.${{ steps.build-image.outputs.cluster-init-version }}.tar - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + artifact_retention_days: ${{ env.ARTIFACT_RETENTION_DAYS }} + test-e2e: runs-on: ubuntu-latest @@ -366,22 +352,6 @@ jobs: - name: Check out code uses: actions/checkout@v2 - - name: Set up Go 1.14 - uses: actions/setup-go@v2 - with: - go-version: "1.14" - - - name: "Setup local k8s cluster" - uses: AbsaOSS/k3d-action@v1.5.0 - with: - cluster-name: ${{ env.CLUSTER_NAME }} - use-default-registry: true - args: >- - --servers 1 - --agents 3 - --port 80:80@loadbalancer - --k3s-server-arg "--no-deploy=traefik,metrics-server" - - name: Download Turing API Docker tar archieve uses: actions/download-artifact@v2 with: @@ -390,14 +360,29 @@ jobs: - name: Download Turing Router Docker tar archieve uses: actions/download-artifact@v2 with: - name: turing-router.${{ needs.build-router.outputs.router-version }}.tar + name: turing-router.${{ env.TURING_ROUTER_VERSION }}.tar - name: Download Cluster Init Docker tar archieve uses: actions/download-artifact@v2 with: - name: cluster-init.${{ needs.build-cluster-init.outputs.cluster-init-version }}.tar + name: cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar + + - name: Set up Go 1.14 + uses: actions/setup-go@v2 + with: + go-version: "1.14" - - name: Publish images to local registry + - name: Run action cluster-init + uses: ./.github/actions/run-cluster-init + with: + cluster_name: ${{ env.CLUSTER_NAME}} + istio_version: ${{ env.ISTIO_VERSION}} + knative_version: ${{ env.KNATIVE_VERSION}} + knative_istio_version: ${{ env.KNATIVE_ISTIO_VERSION}} + local_registry: ${{ env.LOCAL_REGISTRY}} + cluster_init_version: ${{ env.CLUSTER_INIT_VERSION }} + + - name: Publish Turing images to local registry env: DOCKER_REPOSITORY: ${{ env.LOCAL_REGISTRY }}/${{ github.repository }} run: | @@ -415,34 +400,6 @@ jobs: ${{ env.DOCKER_REPOSITORY }}/turing-router:${{ env.TURING_ROUTER_VERSION }} docker push ${{ env.DOCKER_REPOSITORY }}/turing-router:${{ env.TURING_ROUTER_VERSION }} - # Cluster init - docker image load --input cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar - docker tag \ - cluster-init:${{ env.CLUSTER_INIT_VERSION }} \ - ${{ env.DOCKER_REPOSITORY }}/cluster-init:${{ env.CLUSTER_INIT_VERSION }} - docker push ${{ env.DOCKER_REPOSITORY }}/cluster-init:${{ env.CLUSTER_INIT_VERSION }} - - - name: "Install Infrastructure" - run: | - kubectl create ns infrastructure - helm upgrade turing-init infra/charts/turing-init \ - --namespace infrastructure \ - --set image.registry=${{ env.LOCAL_REGISTRY }}/ \ - --set image.repository=${{ github.repository }}/cluster-init \ - --set image.tag=${{ env.CLUSTER_INIT_VERSION }} \ - --set knative.domains="127.0.0.1.nip.io" \ - --set knative.registriesSkippingTagResolving=${{ env.LOCAL_REGISTRY }} \ - --install \ - --wait - - # wait for install infra job to finish - kubectl logs -n infrastructure -f $(kubectl get pod --namespace infrastructure | grep -v 'NAME' | grep -v 'spark' | head -n 1 | awk '{print $1}') - kubectl get pod --all-namespaces - kubectl get svc --all-namespaces - kubectl wait -n infrastructure --for=condition=complete --timeout=10m job/turing-init-spark-operator-webhook-init - # Might fail the first time but the 2nd run should work, rarely fails on the first try though. - kubectl wait -n infrastructure --for=condition=complete --timeout=10m job/turing-init-init - - name: "Install Vault" if: ${{ !matrix.useInClusterConfig }} env: @@ -500,7 +457,8 @@ jobs: run: | kubectl apply -f infra/e2e/turing.mockserver.yaml - - uses: jupyterhub/action-k8s-await-workloads@v1 + - name: Run action await k8 workloads + uses: jupyterhub/action-k8s-await-workloads@v1 id: wait-for-deployment with: workloads: >- @@ -571,10 +529,6 @@ jobs: KUBECONFIG_USE_LOCAL: true run: go test -v -parallel=2 ./e2e/... -tags=e2e -run TestEndToEnd - # Invoke helm hooks on delete - - name: Tear down infrastructure job - run: helm delete --namespace infrastructure turing-init --timeout 15m - release-rules: runs-on: ubuntu-latest outputs: