From 1b6e88cf66682d414ef7a3f95a9c782d506b016d Mon Sep 17 00:00:00 2001 From: Tiago Castro Date: Tue, 24 Dec 2024 16:30:25 +0000 Subject: [PATCH] ci: switch from jenkins to github actions Jenkins has become troublesome to maintain and makes it harder for users to see and test output without adding new users. We could probably spend some time improving that, but the current Jenkins is also setup on systems managed by the sponsoring company so it makes sense to move away from that at least. In order to run the CI tests we need a relatively powerful CI system so we are making use of the ubuntu-latest-16-cores runners. The tests seem to run slower than on Jenkins. Part of it may be explained because the dependencies are not pre-cache and also the pre-requisites which take time to install. Signed-off-by: Tiago Castro --- .github/workflows/helm-chart.yml | 31 +++++++++++++++++++++ .github/workflows/image-pr.yml | 19 +++++++++++++ .github/workflows/image.yml | 29 ++++++++++++++++++++ .github/workflows/k8s-ci.yml | 1 + .github/workflows/lint.yml | 34 +++++++++++++++++++++++ .github/workflows/nightly-ci.yml | 18 +++++++++++++ .github/workflows/pr-ci.yml | 31 +++++++++++++++++++++ .github/workflows/template-chart.yml | 22 --------------- .github/workflows/unit-int.yml | 40 ++++++++++++++++++++++++++++ scripts/rust/test.sh | 28 +++++++++++++++++++ 10 files changed, 231 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/helm-chart.yml create mode 100644 .github/workflows/image-pr.yml create mode 100644 .github/workflows/image.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/nightly-ci.yml create mode 100644 .github/workflows/pr-ci.yml delete mode 100644 .github/workflows/template-chart.yml create mode 100644 .github/workflows/unit-int.yml create mode 100755 scripts/rust/test.sh diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml new file mode 100644 index 000000000..d5a845a07 --- /dev/null +++ b/.github/workflows/helm-chart.yml @@ -0,0 +1,31 @@ +name: Helm Chart CI +on: + workflow_call: + pull_request: + types: ['opened', 'edited', 'reopened', 'synchronize'] + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + helm-chart-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: DeterminateSystems/nix-installer-action@v14 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Pre-populate nix-shell + run: | + export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) + echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV + nix-shell --run "echo" ./scripts/helm/shell.nix + - name: HelmChart publish test + run: | + nix-shell --pure --run "./scripts/helm/test-publish-chart-yaml.sh" ./scripts/helm/shell.nix + - name: HelmChart Readme + run: nix-shell --run "./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix + - name: HelmChart Template + run: nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix \ No newline at end of file diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml new file mode 100644 index 000000000..ac13554fb --- /dev/null +++ b/.github/workflows/image-pr.yml @@ -0,0 +1,19 @@ +name: ImageBuild CI +on: + workflow_call: + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + image-build-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: DeterminateSystems/nix-installer-action@v14 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Test building the release images + run: ./scripts/release.sh --skip-publish --build-bins diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 000000000..056ac71b3 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,29 @@ +name: Image Push +on: + push: + branches: + - develop + - 'release/**' + tags: + - 'v[0-9]+.[0-9]+.[0-9]+**' + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + image-build-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: DeterminateSystems/nix-installer-action@v14 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push the release images + run: ./scripts/release.sh diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index 63f315e69..8f07b65c7 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -2,6 +2,7 @@ name: K8s CI on: pull_request: types: ['opened', 'edited', 'reopened', 'synchronize'] + workflow_call: jobs: k8s-ci: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..57be5eb99 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,34 @@ +name: Unit/Int CI +on: + workflow_call: + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: DeterminateSystems/nix-installer-action@v14 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Pre-populate nix-shell + run: | + export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) + echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV + nix-shell --run "echo" shell.nix + - name: Handle Rust dependencies caching + uses: Swatinem/rust-cache@v2 + - name: Lint rust code + run: | + nix-shell --run "./dependencies/control-plane/scripts/rust/generate-openapi-bindings.sh" + nix-shell --run "./scripts/rust/linter.sh" + - name: Lint python code + run: nix-shell --run "black ./tests/bdd" + - name: Lint nix code + run: nix-shell --run "nixpkgs-fmt ." + - name: Check submodules + run: nix-shell --run "./scripts/git/check-submodule-branches.sh" diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml new file mode 100644 index 000000000..4acfd2aa2 --- /dev/null +++ b/.github/workflows/nightly-ci.yml @@ -0,0 +1,18 @@ +name: Nightly CI +on: + workflow_dispatch: + push: + branches: + - ci + +jobs: + ci: + uses: ./.github/workflows/pr-ci.yml + nightly-ci: + if: ${{ success() }} + needs: + - ci + runs-on: ubuntu-latest + steps: + - name: CI succeeded + run: exit 0 diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml new file mode 100644 index 000000000..8ebca6be8 --- /dev/null +++ b/.github/workflows/pr-ci.yml @@ -0,0 +1,31 @@ +name: Bors CI +on: + workflow_call: + push: + branches: + - staging + - trying + +jobs: + lint-ci: + uses: ./.github/workflows/lint.yml + helm-ci: + uses: ./.github/workflows/helm-chart.yml + int-ci: + uses: ./.github/workflows/unit-int.yml + image-ci: + uses: ./.github/workflows/image-pr.yml + k8s-ci: + uses: ./.github/workflows/k8s-ci.yml + bors-ci: + if: ${{ success() }} + needs: + - lint-ci + - helm-ci + - int-ci + - image-ci + - k8s-ci + runs-on: ubuntu-latest + steps: + - name: CI succeeded + run: exit 0 diff --git a/.github/workflows/template-chart.yml b/.github/workflows/template-chart.yml deleted file mode 100644 index 69d289fb1..000000000 --- a/.github/workflows/template-chart.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Helm Template -on: - pull_request: - types: ['opened', 'edited', 'reopened', 'synchronize'] - push: - branches: - - staging - -jobs: - template-chart: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v22 - - name: Pre-populate nix-shell - run: | - export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) - echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV - nix-shell --pure --run "echo" ./scripts/helm/shell.nix - - name: Check if the chart can be rendered with the default values - run: | - nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix diff --git a/.github/workflows/unit-int.yml b/.github/workflows/unit-int.yml new file mode 100644 index 000000000..df686fe78 --- /dev/null +++ b/.github/workflows/unit-int.yml @@ -0,0 +1,40 @@ +name: Integration CI +on: + workflow_call: + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + int-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + - uses: DeterminateSystems/nix-installer-action@v14 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Pre-populate nix-shell + run: | + export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) + echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV + nix-shell --run "echo" shell.nix + - name: Handle Rust dependencies caching + uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ startsWith(github.ref_name, 'release/') || github.ref_name == 'develop' || github.ref_name == 'ci' }} + - name: Build the binaries + run: nix-shell --run "cargo build --bins" + - name: Build the tests + run: nix-shell --run "./scripts/rust/test.sh --no-run" + - name: Run Tests + run: | + # includes both unit and integration tests + nix-shell --run "./scripts/rust/test.sh" +# debugging + # - name: Setup tmate session + # if: ${{ failure() }} + # timeout-minutes: 120 + # uses: mxschmitt/action-tmate@v3 diff --git a/scripts/rust/test.sh b/scripts/rust/test.sh new file mode 100755 index 000000000..42ead3e4b --- /dev/null +++ b/scripts/rust/test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(dirname "$0")" + +ARGS="" +OPTS="" +DO_ARGS= +while [ "$#" -gt 0 ]; do + case $1 in + --) + DO_ARGS="y" + shift;; + *) + if [ "$DO_ARGS" == "y" ]; then + ARGS="$ARGS $1" + else + OPTS="$OPTS $1" + fi + shift;; + esac +done + +set -euxo pipefail + +# build test dependencies +cargo build --bins + +cargo test ${OPTS} -- ${ARGS} --test-threads=1