check.yml

name: Checks

on:
  pull_request:
    types: [opened, synchronize]
  push:
    branches:
      - main
  merge_group:
    types: [checks_requested]

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

# Uncomment the line below to enable artifacts debugging
# env:
#   ACTIONS_RUNNER_DEBUG: true

jobs:
  notification:
    runs-on: ubuntu-22.04
    timeout-minutes: 1
    if: ${{ !github.event.pull_request.head.repo.fork && github.event.number }}
    permissions:
      pull-requests: write
    steps:
      - uses: actions/checkout@v4

      - uses: marocchino/sticky-pull-request-comment@v2
        with:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}
          message: |
            ### 🪓 The generated code will be pushed at the end of the CI.

            Action triggered by commit [`${{ github.sha }}`](${{ github.event.pull_request.base.repo.html_url }}/commit/${{ github.sha }}).

            _Please do not push any generated code to this pull request._

  setup:
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    steps:
      - name: debugging - dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJSON(github) }}
        run: echo "$GITHUB_CONTEXT"

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup
        id: setup
        uses: ./.github/actions/setup

      # Computing jobs that should run
      - name: Setting diff outputs variables
        id: diff
        shell: bash
        run: |
          previousCommit=${{ github.event.before }}
          baseRef=${{ github.base_ref }}
          origin=$( [[ -z $baseRef ]] && echo $previousCommit || echo "origin/$baseRef" )

          yarn workspace scripts setRunVariables "$origin"

      - name: Compute specs matrix
        id: spec-matrix
        shell: bash
        run: yarn workspace scripts createMatrix ${{ steps.diff.outputs.ORIGIN_BRANCH }}

      - name: Compute the client codegen matrix
        id: gen-matrix
        shell: bash
        run: yarn workspace scripts createMatrix ${{ steps.diff.outputs.ORIGIN_BRANCH }} clients

      - name: check that custom actions are built
        run: |
          yarn workspace scripts build:actions
          diff=$(git status --porcelain ./scripts/ci/actions | wc -l)

          if [[ $diff > 0 ]]; then
            echo "Build the custom github actions by running \`yarn workspace scripts build:actions\`"
          fi

          exit $diff

      - name: Lint GitHub actions
        run: yarn github-actions:lint

      - name: Lint generators
        run: |
          yarn cli format java generators
          diff=$(git status --porcelain ./generators | wc -l)

          if [[ $diff > 0 ]]; then
            echo "Format the generators folder by running 'apic format java generators'"
          fi

          exit $diff

      - name: Lint json files
        run: |
          yarn eslint --ext=json .
          echo 'Use yarn fix:json to fix issues'

    outputs:
      RUN_SCRIPTS: ${{ steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 }}

      SPECS_MATRIX: ${{ steps.spec-matrix.outputs.MATRIX }}

      RUN_GEN: ${{ steps.gen-matrix.outputs.RUN_GEN }}
      GEN_MATRIX: ${{ steps.gen-matrix.outputs.GEN_MATRIX }}
      JAVASCRIPT_DATA: ${{ steps.gen-matrix.outputs.JAVASCRIPT_DATA }}
      RUN_GEN_JAVASCRIPT: ${{ steps.gen-matrix.outputs.RUN_GEN_JAVASCRIPT }}

      SWIFT_DATA: ${{ steps.gen-matrix.outputs.SWIFT_DATA }}
      RUN_MACOS_SWIFT_CTS: ${{ steps.gen-matrix.outputs.RUN_MACOS_SWIFT_CTS }}

      RUN_MACOS_KOTLIN_BUILD: ${{ steps.gen-matrix.outputs.RUN_MACOS_KOTLIN_BUILD }}

  scripts:
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: setup
    if: ${{ needs.setup.outputs.RUN_SCRIPTS == 'true' }}
    steps:
      - uses: actions/checkout@v4

      - name: Setup
        uses: ./.github/actions/setup
        with:
          type: minimal

      - name: Check script linting
        run: yarn scripts:lint

      - name: Test scripts
        run: yarn scripts:test

      - name: Lint custom eslint plugin
        run: yarn workspace eslint-plugin-automation-custom lint

      - name: Test custom eslint plugin
        run: yarn workspace eslint-plugin-automation-custom test

  specs:
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: setup
    steps:
      - uses: actions/checkout@v4

      - name: Setup
        uses: ./.github/actions/setup
        with:
          type: minimal

      - name: Building specs
        run: yarn cli build specs ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX).toRun }}

      - name: Store bundled specs
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          name: specs
          path: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX).bundledPath }}

  client_gen_javascript:
    timeout-minutes: 10
    runs-on: ubuntu-22.04
    needs:
      - setup
      - specs
      - scripts
    if: |
      always() &&
      needs.setup.outputs.RUN_GEN_JAVASCRIPT == 'true' &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'failure')
    env:
      ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
      ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
      MONITORING_API_KEY: ${{ secrets.MONITORING_API_KEY }}
    steps:
      - uses: actions/checkout@v4

      - name: Download specs artifacts
        uses: ./scripts/ci/actions/restore-artifacts
        with:
          type: specs

      - name: Remove generated clients
        if: ${{ startsWith(env.head_ref, 'chore/prepare-release-') }}
        run: |
          cd clients/algoliasearch-client-javascript/packages
          ls | grep -v -E "(client-common|requester-*|algoliasearch)" | xargs rm -rf
          cd algoliasearch
          ls | grep -v -E "(__tests__|jest.config.ts)" | xargs rm -rf

      - name: Setup
        uses: ./.github/actions/setup
        with:
          language: javascript

      - name: Generate clients
        run: yarn cli generate javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}

      - name: Update `yarn.lock` for JavaScript
        run: cd clients/algoliasearch-client-javascript && YARN_ENABLE_HARDENED_MODE=0 YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install

      - name: Build clients
        run: ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).buildCommand }}

      - name: Run common and requester tests
        run: cd clients/algoliasearch-client-javascript && yarn test ${{ !contains(fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun, 'algoliasearch') && '--ignore algoliasearch' || '' }}

      - name: Test JavaScript bundle size
        if: ${{ startsWith(env.head_ref, 'chore/prepare-release-') }}
        run: cd clients/algoliasearch-client-javascript && yarn test:size

      - name: Remove previous CTS output
        run: rm -rf ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).testsToDelete }}

      - name: Generate CTS
        run: yarn cli cts generate javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}

        # JavaScript test deps (needs to be cached because they are huge and inefficient)
      - name: Get yarn js test cache directory path
        shell: bash
        id: yarn-cache-dir-tests
        run: echo "dir=$(cd tests/output/javascript && yarn config get cacheFolder)" >> $GITHUB_OUTPUT

      - name: Restore Yarn js tests
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-tests.outputs.dir || 'tests/output/javascript/.yarn/cache' }}
          key: yarn-cache-tests-${{ hashFiles('tests/output/javascript/yarn.lock') }}

      - name: Cache js tests node modules
        uses: actions/cache@v4
        with:
          path: tests/output/javascript/node_modules
          key: node-modules-tests-${{ hashFiles('tests/output/javascript/yarn.lock') }}

      - name: Run unit CTS
        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --no-e2e

      - name: Run e2e CTS
        id: cts-e2e
        continue-on-error: true
        if: ${{ !github.event.pull_request.head.repo.fork }}
        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --no-client --no-requests

      - name: Retry e2e CTS
        if: ${{ !github.event.pull_request.head.repo.fork && steps.cts-e2e.outcome == 'failure' }}
        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --no-client --no-requests

      - name: Run benchmarks
        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --benchmark --no-client --no-requests --no-e2e

      - name: Generate code snippets for documentation
        run: yarn cli snippets javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}

      - name: Zip artifact before storing
        run: zip -r -y clients-javascript.zip clients/algoliasearch-client-javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).testsToStore }} ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).snippetsToStore }} -x "**/node_modules**" "**/.yarn/cache/**" "**/.yarn/install-state.gz" "**/build/**" "**/dist/**" "**/.gradle/**" "**/bin/**" "**/.nx/**"

      - name: Store javascript clients
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          name: clients-javascript
          path: clients-javascript.zip

  client_gen:
    timeout-minutes: 10
    runs-on: ubuntu-22.04
    needs:
      - setup
      - specs
      - scripts
    if: |
      always() &&
      needs.setup.outputs.RUN_GEN == 'true' &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'failure')
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup.outputs.GEN_MATRIX) }}
    env:
      ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
      ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
      MONITORING_API_KEY: ${{ secrets.MONITORING_API_KEY }}
    steps:
      - uses: actions/checkout@v4

      - name: Download specs artifacts
        uses: ./scripts/ci/actions/restore-artifacts
        with:
          type: specs

      - name: Setup
        uses: ./.github/actions/setup
        with:
          language: ${{ matrix.client.language }}

      - name: Generate clients
        run: yarn cli generate ${{ matrix.client.language }} ${{ matrix.client.toRun }}

      - name: Update composer.lock
        if: ${{ matrix.client.language == 'php' && startsWith(env.head_ref, 'chore/renovateBaseBranch') }}
        run: cd ${{ matrix.client.path }} && composer update

      - name: Check for file duplicates in Swift
        if: ${{ matrix.client.language == 'swift' }}
        run: |
          set -eo pipefail
          cd clients/algoliasearch-client-swift
          find Sources  -type f | rev | cut -d '/' -f1 | rev | sort | uniq -d
          [ $(find Sources  -type f | rev | cut -d '/' -f1 | rev | sort | uniq -d | wc -l) -gt 0 ] && echo "Duplicates found" && exit 1 || echo "No duplicate found"

      - name: Build clients
        run: ${{ matrix.client.buildCommand }}

      - name: Run Java 'algoliasearch' public API validation
        if: ${{ matrix.client.language == 'java' }}
        run: |
          cd ${{ matrix.client.path }}
          ./gradlew :api:japicmp || exit 0

          FILE=${{ matrix.client.path }}/api/build/reports/japi.txt

          if [[ -f "$FILE" ]]; then
            cat $FILE
          fi

      - name: Remove previous CTS output
        run: rm -rf ${{ matrix.client.testsToDelete }} || true

      - name: Generate CTS
        run: yarn cli cts generate ${{ matrix.client.language }} ${{ matrix.client.toRun }}

      - name: Run unit CTS
        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --no-e2e

      - name: Run e2e CTS
        id: cts-e2e
        continue-on-error: true
        if: ${{ !github.event.pull_request.head.repo.fork }}
        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --no-client --no-requests

      - name: Retry e2e CTS
        if: ${{ !github.event.pull_request.head.repo.fork && steps.cts-e2e.outcome == 'failure' }}
        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --no-client --no-requests

      - name: Run benchmarks
        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --benchmark --no-client --no-requests --no-e2e

      - name: Generate code snippets for documentation
        run: yarn cli snippets ${{ matrix.client.language }} ${{ matrix.client.toRun }}

      - name: Zip artifact before storing
        run: zip -r -y clients-${{ matrix.client.language }}.zip ${{ matrix.client.path }} ${{ matrix.client.testsToStore }} ${{ matrix.client.snippetsToStore }} -x "**/node_modules**" "**/__pycache__/**" "**/.yarn/cache/**" "**/build/**" "**/.build/**" "**/dist/**" "**/.gradle/**" "**/bin/**" "**/vendor/**" "**/target/**" "**/.dart_tool/**"

      - name: Store ${{ matrix.client.language }} clients
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          name: clients-${{matrix.client.language }}
          path: clients-${{matrix.client.language }}.zip

  kotlin_build_macos:
    timeout-minutes: 10
    runs-on: macos-latest
    needs:
      - setup
      - client_gen
    if: |
      always() &&
      needs.setup.outputs.RUN_GEN == 'true' &&
      needs.setup.outputs.RUN_MACOS_KOTLIN_BUILD == 'true' &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'failure')
    outputs:
      success: ${{ steps.setoutput.outputs.success }}
    steps:
      - uses: actions/checkout@v4

      - name: Download artifacts
        uses: ./scripts/ci/actions/restore-artifacts
        with:
          type: languages
          languages: |
            kotlin

      - name: Setup
        uses: ./.github/actions/setup
        with:
          type: minimal
          language: kotlin

      - run: yarn cli build clients kotlin

      - name: Set output
        id: setoutput
        run: echo "success=true" >> "$GITHUB_OUTPUT"

  swift_cts_macos:
    timeout-minutes: 20
    runs-on: macos-latest
    needs:
      - setup
      - client_gen
    if: |
      always() &&
      needs.setup.outputs.RUN_GEN == 'true' &&
      needs.setup.outputs.RUN_MACOS_SWIFT_CTS == 'true' &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'failure')
    env:
      ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
      ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
      MONITORING_API_KEY: ${{ secrets.MONITORING_API_KEY }}
    outputs:
      success: ${{ steps.setoutput.outputs.success }}
    steps:
      - uses: actions/checkout@v4

      - name: Download artifacts
        uses: ./scripts/ci/actions/restore-artifacts
        with:
          type: languages
          languages: |
            swift

      - name: Setup
        uses: ./.github/actions/setup
        with:
          type: minimal
          language: swift

      - run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v

      - name: Set output
        id: setoutput
        run: echo "success=true" >> "$GITHUB_OUTPUT"

  codegen:
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    needs:
      - setup
      - client_gen
      - client_gen_javascript
    if: |
      always() &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'failure')
    permissions:
      pull-requests: write
    outputs:
      success: ${{ steps.setoutput.outputs.success }}
      generatedCommit: ${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT || github.event.pull_request.head.ref }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}
          token: ${{ secrets.ALGOLIA_BOT_TOKEN }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Download all artifacts
        uses: ./scripts/ci/actions/restore-artifacts
        with:
          type: all

      - name: Setup
        uses: ./.github/actions/setup
        with:
          type: minimal

      - name: Generate documentation specs with code snippets
        run: yarn cli build specs ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX).toRun }} --docs

      - name: Read benchmark results
        id: benchmark
        run: |
          # merge all benchmark results into a single json, and remove the files
          results=$(jq -s 'add' -c tests/output/**/benchmarkResult.json)
          {
            echo 'BENCHMARK_SECTION<<EOF'
            echo "<details>"
            echo "<summary>📊 Benchmark results</summary>"
            echo "" # empty line is required to make the table work
            echo "Benchmarks performed on the `search` method using a mock server, the results might not reflect the real-world performance."
            # format the json to a markdown table with column "Language" and "rate"
            echo "| Language | Rate |"
            echo "| :------- | ---: |"
            echo "$results" | jq -r 'to_entries | map([.key, .value.rate]) | sort_by(.[1]) | reverse | .[] | @tsv' | awk -F'\t' '{printf "| %-10s | %10d |\n", $1, $2}'
            echo "</details>"
            echo 'EOF'
          } >> "$GITHUB_OUTPUT"
          rm -rf tests/output/**/benchmarkResult.json

      - name: Push generated code
        id: pushGeneratedCode
        run: yarn workspace scripts pushGeneratedCode
        env:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}
          PR_NUMBER: ${{ github.event.number }}

      - name: update generation comment
        uses: marocchino/sticky-pull-request-comment@v2
        if: ${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT == '' }}
        with:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}
          message: |
            ### No code generated

            _If you believe code should've been generated, please, [report the issue](https://github.com/algolia/api-clients-automation/issues/new?assignees=&labels=bug%2Ctriage&projects=&template=Bug_report.yml&title=%5Bbug%5D%3A+)._

            ${{ steps.benchmark.outputs.BENCHMARK_SECTION }}

      - name: update generation comment
        uses: marocchino/sticky-pull-request-comment@v2
        if: ${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT != '' }}
        with:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}
          message: |
            ### ✔️ Code generated!

            |  Name | Link |
            |---------------------------------|------------------------|
            | 🪓 Triggered by | [`${{ github.sha }}`](${{ github.event.pull_request.base.repo.html_url }}/commit/${{ github.sha }}) |
            | 🍃 Generated commit | [`${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT }}`](${{ github.event.pull_request.base.repo.html_url }}/commit/${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT }}) |
            | 🌲 Generated branch | [`generated/${{ github.head_ref }}`](${{ github.event.pull_request.base.repo.html_url }}/tree/generated/${{ github.head_ref }}) |

            ${{ steps.benchmark.outputs.BENCHMARK_SECTION }}

      - name: Build website
        if: ${{ github.ref == 'refs/heads/main' || github.base_ref == 'main' }}
        run: yarn website:build

      - name: Deploy documentation
        if: ${{ github.ref == 'refs/heads/main' || github.base_ref == 'main' }}
        uses: nwtgck/actions-netlify@v3.0
        with:
          publish-dir: 'website/build'
          production-branch: main
          github-token: ${{ secrets.GITHUB_TOKEN }}
          deploy-message: 'Deploy from GitHub Actions'
          enable-pull-request-comment: true
          enable-commit-comment: true
          overwrites-pull-request-comment: true
          netlify-config-path: ./netlify.toml
        env:
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
        timeout-minutes: 5

      - name: Set output
        id: setoutput
        run: echo "success=true" >> "$GITHUB_OUTPUT"

  check_green:
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs:
      - codegen
      - swift_cts_macos
      - kotlin_build_macos
    if: always()
    steps:
      - run: |
          if [[ ("${{ needs.codegen.outputs.success }}" != "true") || ("${{ needs.swift_cts_macos.result }}" != "skipped" && "${{ needs.swift_cts_macos.outputs.success }}" != "true") || ("${{ needs.kotlin_build_macos.result }}" != "skipped" && "${{ needs.kotlin_build_macos.outputs.success }}" != "true")]]; then
            echo "a needed step failed"
            exit 1
          fi

  push_and_release:
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    needs:
      - codegen
      - check_green
    if: |
      always() &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'failure') &&
      github.ref == 'refs/heads/main'
    permissions:
      pull-requests: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ needs.codegen.outputs.generatedCommit }}
          token: ${{ secrets.ALGOLIA_BOT_TOKEN }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Setup
        uses: ./.github/actions/setup
        with:
          type: minimal

      - name: Spread generation to each repository
        id: spreadGeneration
        run: yarn workspace scripts spreadGeneration
        env:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}

      - name: Wait for all release CI
        run: yarn workspace scripts waitForAllReleases ${{ steps.spreadGeneration.outputs.PUSHED_LANGUAGES }}
        env:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}

      - name: Push specs to algolia documentation
        run: yarn workspace scripts pushToAlgoliaDoc
        env:
          GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}