Run deployer tests in separate workflow

.github/workflows/build.yml

@@ -34,30 +34,14 @@ jobs:
           name: flyctl
           path: dist/default_linux_amd64_v1/flyctl
           overwrite: true
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: https://index.docker.io/v1/
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-      - name: Prepare flyctl for deployer Docker build
-        run: |
-          mkdir -p bin
-          cp dist/default_linux_amd64_v1/flyctl bin/flyctl
-          chmod +x bin/flyctl
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          context: .
-          file: deployer.Dockerfile
-          tags: flyio/deployer:${{ github.sha }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
 
   preflight:
     needs: test_build
     uses: ./.github/workflows/preflight.yml
     secrets: inherit
+
+  # deployer-tests:
+  #   needs: test_build
+  #   uses: ./.github/workflows/deployer-tests.yml
+  #   secrets: inherit
+

.github/workflows/deployer-tests.yml

@@ -0,0 +1,124 @@
+name: Deployer tests
+on:
+  push:
+
+jobs:
+  build-deployer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+          check-latest: true
+      - name: "Build flyctl"
+        run: make build
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: https://index.docker.io/v1/
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Upload flyctl for the deployer tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: deployer-flyctl
+          path: bin/flyctl
+          overwrite: true
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          context: .
+          file: deployer.Dockerfile
+          tags: flyio/deployer:${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  deployer-tests:
+    needs: build-deployer
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        vm_size: [""]
+        parallelism: [10]
+        index: [0,1,2,3,4,5,6,7,8,9]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+          check-latest: true
+      - name: Get go version
+        id: go-version
+        run: echo "name=version::$(go env GOVERSION)" >> $GITHUB_OUTPUT
+      - name: Install gotesplit, set FLY_PREFLIGHT_TEST_APP_PREFIX
+        run: |
+          curl -sfL https://raw.githubusercontent.com/Songmu/gotesplit/v0.2.1/install.sh | sh -s
+          echo "FLY_PREFLIGHT_TEST_APP_PREFIX=pf-gha-$(openssl rand -hex 4)" >> "$GITHUB_ENV"
+      - uses: actions/download-artifact@v4
+        with:
+          name: deployer-flyctl
+          path: deployer-bin
+      - name: Move flyctl binary to correct directory
+        run: |
+          mkdir -p bin
+          mv deployer-bin/flyctl bin/flyctl
+          chmod +x bin/flyctl
+      - name: Run deployer tests
+        id: deployer-tests
+        env:
+          FLY_PREFLIGHT_TEST_ACCESS_TOKEN: ${{ secrets.FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN }}
+          FLY_PREFLIGHT_TEST_FLY_ORG: flyctl-ci-preflight
+          # This VM size is only available in ORD.
+          FLY_PREFLIGHT_TEST_FLY_REGIONS: ord
+          FLY_PREFLIGHT_TEST_NO_PRINT_HISTORY_ON_FAIL: "true"
+          FLY_FORCE_TRACE: "true"
+          FLY_PREFLIGHT_TEST_VM_SIZE: ${{ matrix.vm_size }}
+          FLY_DEPLOYER_IMAGE: "flyio/deployer:${{ github.sha }}"
+          FLY_PREFLIGHT_TEST_APP_PREFIX: "deployertest"
+        run: |
+          export PATH=$PWD/bin:$PATH
+          echo -n failed= >> $GITHUB_OUTPUT
+          ./scripts/deployer-tests.sh -r "${{ github.ref }}" -t "${{ matrix.parallelism }}" -i "${{ matrix.index }}" -o $GITHUB_OUTPUT
+      - name: Post failure to slack
+        if: ${{ github.ref == 'refs/heads/master' && failure() }}
+        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.PREFLIGHT_SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+        with:
+          channel-id: 'C0790M2E0G2'
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":sob: deployer tests failed: ${{ steps.deployer-tests.outputs.failed }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                  }
+                }
+              ]
+            }
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+        if: always()
+      - name: Clean up any un-deleted deployer-tests apps
+        if: always()
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN }}
+          FLY_PREFLIGHT_TEST_FLY_ORG: flyctl-ci-preflight
+          FLY_PREFLIGHT_TEST_APP_PREFIX: "deployertest"
+        run: |
+          ./scripts/delete_preflight_apps.sh "$FLY_PREFLIGHT_TEST_APP_PREFIX"

.github/workflows/preflight.yml

@@ -51,7 +51,7 @@ jobs:
           FLY_PREFLIGHT_TEST_NO_PRINT_HISTORY_ON_FAIL: "true"
           FLY_FORCE_TRACE: "true"
           FLY_PREFLIGHT_TEST_VM_SIZE: ${{ matrix.vm_size }}
-          FLY_DEPLOYER_IMAGE: "flyio/deployer:${{ github.sha }}"
+          FLY_PREFLIGHT_TEST_APP_PREFIX: "preflight"
         run: |
           export PATH=$PWD/bin:$PATH
           echo -n failed= >> $GITHUB_OUTPUT
@@ -82,5 +82,6 @@ jobs:
         env:
           FLY_API_TOKEN: ${{ secrets.FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN }}
           FLY_PREFLIGHT_TEST_FLY_ORG: flyctl-ci-preflight
+          FLY_PREFLIGHT_TEST_APP_PREFIX: "preflight"
         run: |
           ./scripts/delete_preflight_apps.sh "$FLY_PREFLIGHT_TEST_APP_PREFIX"

deploy.rb

@@ -279,7 +279,7 @@
   else
     image_ref = "registry.fly.io/#{APP_NAME}:#{image_tag}"
 
-    exec_capture("flyctl deploy --build-only --depot=false --push -a #{APP_NAME} --image-label #{image_tag}")
+    exec_capture("flyctl deploy --build-only --push -a #{APP_NAME} --image-label #{image_tag}")
     artifact Artifact::DOCKER_IMAGE, { ref: image_ref }
     image_ref
   end

scripts/deployer-tests.sh

@@ -0,0 +1,60 @@
+#! /bin/bash
+set -euo pipefail
+
+ref=
+total=
+index=
+out=
+
+while getopts r:t:i:o: name
+do
+    case "$name" in
+        r)
+	    ref="$OPTARG"
+	    ;;
+        t)
+	    total="$OPTARG"
+	    ;;
+        i)
+	    index="$OPTARG"
+	    ;;
+        o)
+	    out="$OPTARG"
+	    ;;
+        ?)
+	    printf "Usage: %s: [-r REF] [-t TOTAL] [-i INDEX] [-o FILE]\n" $0
+            exit 2
+	    ;;
+    esac
+done
+
+shift $(($OPTIND - 1))
+
+test_opts=
+if [[ "$ref" != "refs/heads/master" ]]; then
+    test_opts=-short
+fi
+
+test_log="$(mktemp)"
+function finish {
+  rm "$test_log"
+}
+trap finish EXIT
+
+set +e
+
+gotesplit \
+    -total "$total" \
+    -index "$index" \
+    github.com/superfly/flyctl/test/deployer/... \
+    -- --tags=integration -v -timeout=10m $test_opts | tee "$test_log"
+test_status=$?
+
+set -e
+
+if [[ -n "$out" ]]; then
+    awk '/^--- FAIL:/{ printf("%s ", $3) }' "$test_log" >> "$out"
+    echo >> "$out"
+fi
+
+exit $test_status

scripts/preflight.sh

@@ -46,7 +46,7 @@ set +e
 gotesplit \
     -total "$total" \
     -index "$index" \
-    github.com/superfly/flyctl/test/preflight/... github.com/superfly/flyctl/test/deployer/... \
+    github.com/superfly/flyctl/test/preflight/... \
     -- --tags=integration -v -timeout=10m $test_opts | tee "$test_log"
 test_status=$?