Update beam to 2.60 (#5516) #2111
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file was automatically generated by sbt-github-actions using the | |
| # githubWorkflowGenerate task. You should add and commit this file to | |
| # your git repository. It goes without saying that you shouldn't edit | |
| # this file by hand! Instead, if you wish to make changes, you should | |
| # change your sbt build configuration to revise the workflow description | |
| # to meet your needs, then regenerate this file. | |
| name: Continuous Integration | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| tags: [v*] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| concurrency: | |
| group: ${{ github.workflow }} @ ${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| name: Build and Test | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13, 2.12] | |
| java: [corretto@11, corretto@17, corretto@21] | |
| exclude: | |
| - scala: 2.12 | |
| java: corretto@17 | |
| - scala: 2.12 | |
| java: corretto@21 | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Install sbt | |
| if: contains(runner.os, 'macos') | |
| run: brew install sbt | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@17) | |
| id: setup-java-corretto-17 | |
| if: matrix.java == 'corretto@17' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 17 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@21) | |
| id: setup-java-corretto-21 | |
| if: matrix.java == 'corretto@21' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 21 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: gcloud auth | |
| if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| export_environment_variables: true | |
| create_credentials_file: true | |
| - name: Setup GitHub Action | |
| run: scripts/gha_setup.sh | |
| - name: Check that workflows are up to date | |
| run: sbt githubWorkflowCheck | |
| - name: Check headers and formatting | |
| if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest' | |
| run: sbt '++ ${{ matrix.scala }}' headerCheckAll javafmtCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck | |
| - name: Test | |
| run: sbt '++ ${{ matrix.scala }}' test | |
| - name: Check binary compatibility | |
| if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest' | |
| run: sbt '++ ${{ matrix.scala }}' mimaReportBinaryIssues | |
| - name: Generate API documentation | |
| if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest' | |
| run: sbt '++ ${{ matrix.scala }}' doc | |
| - name: Check dependencies | |
| run: sbt '++ ${{ matrix.scala }}' undeclaredCompileDependenciesTest unusedCompileDependenciesTest | |
| - name: Make target directories | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| run: mkdir -p scio-bom/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-test/parquet/target scio-test/core/target scio-google-cloud-platform/target scio-smb/target scio-test/google-cloud-platform/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target | |
| - name: Compress target directories | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| run: tar cf targets.tar scio-bom/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-test/parquet/target scio-test/core/target scio-google-cloud-platform/target scio-smb/target scio-test/google-cloud-platform/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target | |
| - name: Upload target directories | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: target-${{ matrix.os }}-${{ matrix.java }}-${{ matrix.scala }} | |
| path: targets.tar | |
| publish: | |
| name: Publish Artifacts | |
| needs: [build] | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Install sbt | |
| if: contains(runner.os, 'macos') | |
| run: brew install sbt | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@17) | |
| id: setup-java-corretto-17 | |
| if: matrix.java == 'corretto@17' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 17 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@21) | |
| id: setup-java-corretto-21 | |
| if: matrix.java == 'corretto@21' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 21 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Download target directories (2.13) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: target-${{ matrix.os }}-${{ matrix.java }}-2.13 | |
| - name: Inflate target directories (2.13) | |
| run: | | |
| tar xf targets.tar | |
| rm targets.tar | |
| - name: Download target directories (2.12) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: target-${{ matrix.os }}-${{ matrix.java }}-2.12 | |
| - name: Inflate target directories (2.12) | |
| run: | | |
| tar xf targets.tar | |
| rm targets.tar | |
| - name: Import signing key | |
| if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == '' | |
| env: | |
| PGP_SECRET: ${{ secrets.PGP_SECRET }} | |
| PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} | |
| run: echo $PGP_SECRET | base64 -d -i - | gpg --import | |
| - name: Import signing key and strip passphrase | |
| if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != '' | |
| env: | |
| PGP_SECRET: ${{ secrets.PGP_SECRET }} | |
| PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} | |
| run: | | |
| echo "$PGP_SECRET" | base64 -d -i - > /tmp/signing-key.gpg | |
| echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg | |
| (echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1) | |
| - name: Package repl | |
| run: sbt scio-repl/assembly | |
| - name: Publish | |
| env: | |
| SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} | |
| SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} | |
| SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }} | |
| run: sbt tlCiRelease | |
| - name: Upload Repl | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: scio-repl/target/scala-2.13/scio-repl.jar | |
| draft: true | |
| dependency-submission: | |
| name: Submit Dependencies | |
| if: github.event.repository.fork == false && github.event_name != 'pull_request' | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Install sbt | |
| if: contains(runner.os, 'macos') | |
| run: brew install sbt | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@17) | |
| id: setup-java-corretto-17 | |
| if: matrix.java == 'corretto@17' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 17 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Setup Java (corretto@21) | |
| id: setup-java-corretto-21 | |
| if: matrix.java == 'corretto@21' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 21 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Submit Dependencies | |
| uses: scalacenter/sbt-dependency-submission@v2 | |
| with: | |
| modules-ignore: scio_2.13 scio_2.12 scio-examples_2.13 scio-examples_2.12 integration_2.13 integration_2.12 scio-jmh_2.13 scio-jmh_2.12 | |
| configs-ignore: test scala-tool scala-doc-tool test-internal | |
| validate-steward: | |
| name: Validate Steward Config | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| java: [temurin@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (fast) | |
| uses: actions/checkout@v4 | |
| - name: Setup Java (temurin@11) | |
| id: setup-java-temurin-11 | |
| if: matrix.java == 'temurin@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 11 | |
| - uses: coursier/setup-action@v1 | |
| with: | |
| apps: scala-steward | |
| - run: scala-steward validate-repo-config .scala-steward.conf | |
| coverage: | |
| name: Test Coverage | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Check that workflows are up to date | |
| run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck | |
| - name: Test coverage | |
| run: sbt '++ ${{ matrix.scala }}' coverage test coverageAggregate | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| avro-legacy: | |
| name: Test Legacy Avro | |
| if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Test | |
| env: | |
| JAVA_OPTS: '-Davro.version=1.8.2' | |
| run: sbt '++ ${{ matrix.scala }}' scio-avro/test | |
| it-test: | |
| name: Integration Test | |
| if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Check that workflows are up to date | |
| run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck | |
| - name: gcloud auth | |
| if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| export_environment_variables: true | |
| create_credentials_file: true | |
| - name: Setup GitHub Action | |
| env: | |
| BQ_READ_TIMEOUT: 30000 | |
| CLOUDSQL_SQLSERVER_PASSWORD: ${{ secrets.CLOUDSQL_SQLSERVER_PASSWORD }} | |
| run: scripts/gha_setup.sh | |
| - name: Test | |
| run: 'sbt ''++ ${{ matrix.scala }}'' ''set integration/test/skip := false'' integration/test' | |
| site: | |
| name: Generate Site | |
| if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| scala: [2.13] | |
| java: [corretto@11] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout current branch (full) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java (corretto@11) | |
| id: setup-java-corretto-11 | |
| if: matrix.java == 'corretto@11' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: 11 | |
| cache: sbt | |
| - name: sbt update | |
| if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false' | |
| run: sbt +update | |
| - name: Check that workflows are up to date | |
| run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck | |
| - name: gcloud auth | |
| if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_CREDENTIALS }} | |
| export_environment_variables: true | |
| create_credentials_file: true | |
| - name: Setup GitHub Action | |
| run: scripts/gha_setup.sh | |
| - name: Generate site | |
| env: | |
| SOCCO: true | |
| run: sbt '++ ${{ matrix.scala }}' scio-examples/compile site/makeSite | |
| - name: Publish site | |
| if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') | |
| uses: peaceiris/actions-gh-pages@v3.9.3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: site/target/site | |
| keep_files: true |