From b308821595cfd7204254f7d9101502103b39b449 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 26 Feb 2024 20:00:33 +0100 Subject: [PATCH 1/5] ci(e2e): set exact commit sha to use for e2e images Resolves #1146 --- .github/workflows/e2e-k3d.yml | 2 +- deploy.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k3d.yml b/.github/workflows/e2e-k3d.yml index da3184b55..43779b3ca 100644 --- a/.github/workflows/e2e-k3d.yml +++ b/.github/workflows/e2e-k3d.yml @@ -96,7 +96,7 @@ jobs: - name: Deploy with helm uses: WyriHaximus/github-action-helm3@v3 with: - exec: ./deploy.py helm --branch ${{ github.ref_name }} --dockerconfigjson ${{ secrets.GHCR_DOCKER_CONFIG }} + exec: ./deploy.py helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} --dockerconfigjson ${{ secrets.GHCR_DOCKER_CONFIG }} - name: Wait for the pods to be ready run: ./.github/scripts/wait_for_pods_to_be_ready.py diff --git a/deploy.py b/deploy.py index de9ef50db..e94e011cc 100755 --- a/deploy.py +++ b/deploy.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 import argparse +import os import subprocess import time from pathlib import Path -import os + import yaml script_path = Path(__file__).resolve() @@ -38,6 +39,7 @@ helm_parser.add_argument('--dev', action='store_true', help='Set up a development environment for running the website and the backend locally') helm_parser.add_argument('--branch', help='Set the branch to deploy with the Helm chart') +helm_parser.add_argument('--sha', help='Set the commit sha to deploy with the Helm chart') helm_parser.add_argument('--dockerconfigjson', help='Set the base64 encoded dockerconfigjson secret for pulling the images') helm_parser.add_argument('--uninstall', action='store_true', help='Uninstall installation') @@ -141,6 +143,9 @@ def handle_helm(): '--set', f"dockerconfigjson={docker_config_json}", ] + if args.sha: + parameters += ['--set', f"sha={args.sha[:7]}"] + if args.dev: parameters += ['--set', "disableBackend=true"] parameters += ['--set', "disableWebsite=true"] From df104c5d925b185a1dd0c5eb2ba41bdc4353fbca Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 26 Feb 2024 20:05:27 +0100 Subject: [PATCH 2/5] We should also wait for the exact sha workflow --- .github/workflows/e2e-k3d.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k3d.yml b/.github/workflows/e2e-k3d.yml index 43779b3ca..6f106cdce 100644 --- a/.github/workflows/e2e-k3d.yml +++ b/.github/workflows/e2e-k3d.yml @@ -82,14 +82,14 @@ jobs: - name: Wait for Backend Docker Image uses: lewagon/wait-on-check-action@v1.3.3 with: - ref: ${{ github.ref }} + ref: ${{ github.sha }} check-name: Build Backend Docker Image repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Wait for Website Docker Image uses: lewagon/wait-on-check-action@v1.3.3 with: - ref: ${{ github.ref }} + ref: ${{ github.sha }} check-name: Build Website Docker Image repo-token: ${{ secrets.GITHUB_TOKEN }} From ee122f3569f685fdaa31a6b85fa2cf8066774e86 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 26 Feb 2024 20:08:41 +0100 Subject: [PATCH 3/5] fix(ci): I used the wrong workflow here 2 weeks ago --- .github/workflows/e2e-k3d.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-k3d.yml b/.github/workflows/e2e-k3d.yml index 6f106cdce..084b73cc3 100644 --- a/.github/workflows/e2e-k3d.yml +++ b/.github/workflows/e2e-k3d.yml @@ -9,7 +9,7 @@ on: - "kubernetes/**" - "preprocessing/**" - "deploy.py" - - ".github/workflows/e2e-test.yml" + - ".github/workflows/e2e-k3d.yml" concurrency: group: ci-${{ github.ref }}-e2e-k3d From 7c6f43319516e573d16f7f8939b0438273625aed Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 26 Feb 2024 20:13:59 +0100 Subject: [PATCH 4/5] feat(dev): add `--verbose` flag to deploy.py to print commands that are executed in shell --- deploy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy.py b/deploy.py index e94e011cc..9032cc81f 100755 --- a/deploy.py +++ b/deploy.py @@ -29,6 +29,7 @@ parser = argparse.ArgumentParser(description='Manage k3d cluster and helm installations.') subparsers = parser.add_subparsers(dest='subcommand', required=True, help='Subcommands') parser.add_argument('--dry-run', action='store_true', help='Print commands instead of executing them') +parser.add_argument('--verbose', action='store_true', help='Print commands that are executed') cluster_parser = subparsers.add_parser('cluster', help='Start the k3d cluster') cluster_parser.add_argument('--dev', action='store_true', @@ -55,11 +56,12 @@ args = parser.parse_args() def run_command(command: list[str], **kwargs): - if args.dry_run: + if args.dry_run or args.verbose: if isinstance(command, str): print(command) else: print(" ".join(map(str,command))) + if args.dry_run: return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") else: return subprocess.run(command, **kwargs) From a6f79f39592e5f68b226e2a027bcc8849724abc1 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 26 Feb 2024 20:14:46 +0100 Subject: [PATCH 5/5] feat(ci): enable command logging for deploy.py in ci --- .github/workflows/e2e-k3d.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k3d.yml b/.github/workflows/e2e-k3d.yml index 084b73cc3..7a41813c8 100644 --- a/.github/workflows/e2e-k3d.yml +++ b/.github/workflows/e2e-k3d.yml @@ -34,7 +34,7 @@ jobs: - name: Create k3d cluster run: | - ./deploy.py cluster + ./deploy.py --verbose cluster - name: Test helm template uses: WyriHaximus/github-action-helm3@v3 @@ -96,7 +96,7 @@ jobs: - name: Deploy with helm uses: WyriHaximus/github-action-helm3@v3 with: - exec: ./deploy.py helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} --dockerconfigjson ${{ secrets.GHCR_DOCKER_CONFIG }} + exec: ./deploy.py --verbose helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} --dockerconfigjson ${{ secrets.GHCR_DOCKER_CONFIG }} - name: Wait for the pods to be ready run: ./.github/scripts/wait_for_pods_to_be_ready.py