From 8ad69dd253f3f7c1da215e0be7cc2741616e5349 Mon Sep 17 00:00:00 2001 From: Kyle Loveless Date: Tue, 31 Oct 2023 15:24:27 -0400 Subject: [PATCH] An attempt, with help from https://github.com/pulumi/pulumi-kubernetes, on starting to deploy java SDK during pre-release, but using https://github.com/pulumi/pulumi-package-publisher/tree/main to reduce duplication. There are a couple open questions: * I was getting complaints about the tree not being clean when building the java version. Not sure why it is different than the other builds. * Tests are not enabled for java, since I believe it requires writing some test like for the other languages. Can take a look, but have not yet. Other notes: * I've delete master.yml, since it seems unused, and is one less place to copy similar logic. * I've updated prelease.yml and release.yml, but not much on main.yml, since it surprised me it did publishing of sdks. Open to discussion on what should be in it. --- .github/workflows/main.yml | 1 + .github/workflows/master.yml | 395 ------------------------------- .github/workflows/prerelease.yml | 35 ++- .github/workflows/release.yml | 28 +++ 4 files changed, 62 insertions(+), 397 deletions(-) delete mode 100644 .github/workflows/master.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e1c7829..bfd0a1b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -349,6 +349,7 @@ jobs: - python - dotnet - go + # TODO: Add java here. But it needs to have tests to run. nodeversion: - 14.x pythonversion: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml deleted file mode 100644 index 41bde2d..0000000 --- a/.github/workflows/master.yml +++ /dev/null @@ -1,395 +0,0 @@ -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - NUGET_PUBLISH_KEY: ${{ secrets.NUGET_PUBLISH_KEY }} - PROVIDER: vantage - PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} - PULUMI_API: https://api.pulumi.com - PULUMI_GO_DEP_ROOT: ${{ github.workspace }}/.. - PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget - PULUMI_TEST_OWNER: "lbrlabs" - PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - PYPI_USERNAME: "__token__" - TRAVIS_OS_NAME: linux -jobs: - build_sdk: - name: build_sdk - needs: prerequisites - runs-on: ubuntu-latest - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Checkout Scripts Repo - uses: actions/checkout@v3 - with: - path: ci-scripts - repository: jaxxstorm/scripts - ref: third_party - - name: Unshallow clone for tags - run: git fetch --prune --unshallow --tags - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{matrix.goversion}} - - name: Install pulumictl - uses: jaxxstorm/action-install-gh-release@v1.9.0 - with: - repo: pulumi/pulumictl - - name: Install Pulumi CLI - uses: pulumi/action-install-pulumi-cli@v2 - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: ${{matrix.nodeversion}} - registry-url: https://registry.npmjs.org - - name: Setup DotNet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: ${{matrix.dotnetversion}} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{matrix.pythonversion}} - - name: Setup Java - uses: actions/setup-java@v3 - with: - cache: gradle - distribution: temurin - java-version: ${{matrix.javaversion}} - - name: Download provider + tfgen binaries - uses: actions/download-artifact@v3 - with: - name: ${{ env.PROVIDER }}-provider.tar.gz - path: ${{ github.workspace }}/bin - - name: Untar provider binaries - run: |- - tar -zxf ${{ github.workspace }}/bin/provider.tar.gz -C ${{ github.workspace}}/bin - find ${{ github.workspace }} -name "pulumi-*-${{ env.PROVIDER }}" -print -exec chmod +x {} \; - - name: Install plugins - run: make install_plugins - - name: Update path - run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - name: Build SDK - run: make build_${{ matrix.language }} - - name: Check worktree clean - run: ./ci-scripts/ci/check-worktree-is-clean - - name: Compress SDK folder - run: tar -zcf sdk/${{ matrix.language }}.tar.gz -C sdk/${{ matrix.language }} - . - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.language }}-sdk.tar.gz - path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz - strategy: - fail-fast: true - matrix: - dotnetversion: - - 3.1.301 - goversion: - - 1.20.x - javaversion: - - "11" - language: - - nodejs - - python - - dotnet - - go - nodeversion: - - 14.x - pythonversion: - - "3.7" - prerequisites: - name: prerequisites - runs-on: ubuntu-latest - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Checkout Scripts Repo - uses: actions/checkout@v3 - with: - path: ci-scripts - repository: jaxxstorm/scripts - ref: third_party - - name: Unshallow clone for tags - run: git fetch --prune --unshallow --tags - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{matrix.goversion}} - - name: Install pulumictl - uses: jaxxstorm/action-install-gh-release@v1.9.0 - with: - repo: pulumi/pulumictl - - name: Install Pulumi CLI - uses: pulumi/action-install-pulumi-cli@v2 - - if: github.event_name == 'pull_request' - name: Install Schema Tools - uses: jaxxstorm/action-install-gh-release@v1.9.0 - with: - repo: mikhailshilkov/schema-tools - - name: Build tfgen & provider binaries - run: make provider - # - if: github.event_name == 'pull_request' - # name: Check Schema is Valid - # run: |- - # echo 'SCHEMA_CHANGES<> $GITHUB_ENV - # schema-tools compare ${{ env.PROVIDER }} master --local-path=provider/cmd/pulumi-resource-${{ env.PROVIDER }}/schema.json >> $GITHUB_ENV - # echo 'EOF' >> $GITHUB_ENV - # - if: github.event_name == 'pull_request' - # name: Comment on PR with Details of Schema Check - # uses: thollander/actions-comment-pull-request@1.0.1 - # with: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # message: | - # ### Does the PR have any schema changes? - - # ${{ env.SCHEMA_CHANGES }} - - name: Tar provider binaries - run: tar -zcf ${{ github.workspace }}/bin/provider.tar.gz -C ${{ github.workspace - }}/bin/ pulumi-resource-${{ env.PROVIDER }} pulumi-tfgen-${{ env.PROVIDER - }} - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ env.PROVIDER }}-provider.tar.gz - path: ${{ github.workspace }}/bin/provider.tar.gz - strategy: - fail-fast: true - matrix: - dotnetversion: - - 3.1.301 - goversion: - - 1.20.x - nodeversion: - - 14.x - pythonversion: - - "3.7" - publish: - name: publish - needs: test - runs-on: ubuntu-latest - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Unshallow clone for tags - run: git fetch --prune --unshallow --tags - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{matrix.goversion}} - - name: Install pulumictl - uses: jaxxstorm/action-install-gh-release@v1.9.0 - with: - repo: pulumi/pulumictl - - name: Install Pulumi CLI - uses: pulumi/action-install-pulumi-cli@v2 - - - name: Set PreRelease Version - run: echo "GORELEASER_CURRENT_TAG=v$(pulumictl get version --language generic)" >> $GITHUB_ENV - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 - with: - args: -p 3 -f .goreleaser.prerelease.yml --rm-dist --skip-validate --timeout - 60m0s - version: latest - strategy: - fail-fast: true - matrix: - dotnetversion: - - 3.1.301 - goversion: - - 1.20.x - nodeversion: - - 14.x - pythonversion: - - "3.7" - publish_sdk: - name: publish_sdk - needs: publish - runs-on: ubuntu-latest - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Checkout Scripts Repo - uses: actions/checkout@v3 - with: - path: ci-scripts - repository: jaxxstorm/scripts - ref: third_party - - name: Unshallow clone for tags - run: git fetch --prune --unshallow --tags - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{matrix.goversion}} - - name: Install pulumictl - uses: jaxxstorm/action-install-gh-release@v1.9.0 - with: - repo: pulumi/pulumictl - - name: Install Pulumi CLI - uses: pulumi/action-install-pulumi-cli@v2 - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: ${{matrix.nodeversion}} - registry-url: https://registry.npmjs.org - - name: Setup DotNet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: ${{matrix.dotnetversion}} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{matrix.pythonversion}} - - name: Setup Java - uses: actions/setup-java@v3 - with: - cache: gradle - distribution: temurin - java-version: ${{matrix.javaversion}} - - name: Download python SDK - uses: actions/download-artifact@v3 - with: - name: python-sdk.tar.gz - path: ${{ github.workspace}}/sdk/ - - name: Uncompress python SDK - run: tar -zxf ${{github.workspace}}/sdk/python.tar.gz -C ${{github.workspace}}/sdk/python - - name: Download dotnet SDK - uses: actions/download-artifact@v3 - with: - name: dotnet-sdk.tar.gz - path: ${{ github.workspace}}/sdk/ - - name: Uncompress dotnet SDK - run: tar -zxf ${{github.workspace}}/sdk/dotnet.tar.gz -C ${{github.workspace}}/sdk/dotnet - - name: Download nodejs SDK - uses: actions/download-artifact@v3 - with: - name: nodejs-sdk.tar.gz - path: ${{ github.workspace}}/sdk/ - - name: Uncompress nodejs SDK - run: tar -zxf ${{github.workspace}}/sdk/nodejs.tar.gz -C ${{github.workspace}}/sdk/nodejs - - run: python -m pip install pip twine - - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - name: Publish SDKs - run: ./ci-scripts/ci/publish-tfgen-package ${{ github.workspace }} - strategy: - fail-fast: true - matrix: - dotnetversion: - - 3.1.301 - javaversion: - - "11" - goversion: - - 1.20.x - nodeversion: - - 14.x - pythonversion: - - "3.7" - test: - name: test - needs: build_sdk - runs-on: ubuntu-latest - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Checkout Scripts Repo - uses: actions/checkout@v3 - with: - path: ci-scripts - repository: jaxxstorm/scripts - ref: third_party - - name: Unshallow clone for tags - run: git fetch --prune --unshallow --tags - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{matrix.goversion}} - - name: Install pulumictl - uses: jaxxstorm/action-install-gh-release@v1.9.0 - with: - repo: pulumi/pulumictl - - name: Install Pulumi CLI - uses: pulumi/action-install-pulumi-cli@v2 - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: ${{matrix.nodeversion}} - registry-url: https://registry.npmjs.org - - name: Setup DotNet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: ${{matrix.dotnetversion}} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{matrix.pythonversion}} - - name: Setup Java - uses: actions/setup-java@v3 - with: - cache: gradle - distribution: temurin - java-version: ${{matrix.javaversion}} - - name: Download provider + tfgen binaries - uses: actions/download-artifact@v3 - with: - name: ${{ env.PROVIDER }}-provider.tar.gz - path: ${{ github.workspace }}/bin - - name: Untar provider binaries - run: |- - tar -zxf ${{ github.workspace }}/bin/provider.tar.gz -C ${{ github.workspace}}/bin - find ${{ github.workspace }} -name "pulumi-*-${{ env.PROVIDER }}" -print -exec chmod +x {} \; - - run: dotnet nuget add source ${{ github.workspace }}/nuget - - name: Download SDK - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.language }}-sdk.tar.gz - path: ${{ github.workspace}}/sdk/ - - name: Uncompress SDK folder - run: tar -zxf ${{ github.workspace }}/sdk/${{ matrix.language }}.tar.gz -C ${{ - github.workspace }}/sdk/${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - name: Install Python deps - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Run tests - run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . - strategy: - fail-fast: true - matrix: - dotnetversion: - - 3.1.301 - goversion: - - 1.20.x - javaversion: - - "11" - language: - - nodejs - - python - - dotnet - - go - nodeversion: - - 14.x - pythonversion: - - "3.7" - -permissions: - contents: write - -name: master -"on": - push: - branches: - - master - paths-ignore: - - CHANGELOG.md - tags-ignore: - - v* - - sdk/* - - '**' diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 3edb0ad..1259b8d 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -12,6 +12,12 @@ env: PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} PYPI_USERNAME: "__token__" TRAVIS_OS_NAME: linux + PUBLISH_REPO_USERNAME: ${{ secrets.OSSRH_USERNAME }} + PUBLISH_REPO_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + SIGNING_KEY_ID: ${{ secrets.JAVA_SIGNING_KEY_ID }} + SIGNING_KEY: ${{ secrets.JAVA_SIGNING_KEY }} + SIGNING_PASSWORD: ${{ secrets.JAVA_SIGNING_PASSWORD }} + JAVAVERSION: "11" jobs: build_sdk: name: build_sdk @@ -51,6 +57,16 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{matrix.pythonversion}} + - name: Setup Java + uses: actions/setup-java@v3 + with: + java-version: ${{ env.JAVAVERSION }} + distribution: temurin + cache: gradle + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + gradle-version: "7.6" - name: Download provider + tfgen binaries uses: actions/download-artifact@v3 with: @@ -66,8 +82,11 @@ jobs: run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - name: Build SDK run: make build_${{ matrix.language }} - - name: Check worktree clean - run: ./ci-scripts/ci/check-worktree-is-clean + # TODO: DO NOT SUBMIT. This fails with modified: + # sdk/java/build/libs/com.pulumi.cockroach.jar + # sdk/java/build/tmp/compileJava/previous-compilation-data.bin + #- name: Check worktree clean + # run: ./ci-scripts/ci/check-worktree-is-clean - name: Compress SDK folder run: tar -zcf sdk/${{ matrix.language }}.tar.gz -C sdk/${{ matrix.language }} . @@ -88,6 +107,7 @@ jobs: - python - dotnet - go + - java nodeversion: - 14.x pythonversion: @@ -272,6 +292,16 @@ jobs: - 14.x pythonversion: - "3.7" + publish_java_sdk: + runs-on: ubuntu-latest + continue-on-error: true + needs: publish + name: publish_java_sdk + steps: + - name: Publish Java SDK + uses: pulumi/pulumi-package-publisher@0058a106b68d8277f17bbea0cd29b2ff6e671adc + with: + sdk: java test: name: test needs: build_sdk @@ -350,6 +380,7 @@ jobs: - python - dotnet - go + # TODO: Add java here. But it needs to have tests to run. nodeversion: - 14.x pythonversion: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d77164..8bca6fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,12 @@ env: PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} PYPI_USERNAME: "__token__" TRAVIS_OS_NAME: linux + PUBLISH_REPO_USERNAME: ${{ secrets.OSSRH_USERNAME }} + PUBLISH_REPO_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + SIGNING_KEY_ID: ${{ secrets.JAVA_SIGNING_KEY_ID }} + SIGNING_KEY: ${{ secrets.JAVA_SIGNING_KEY }} + SIGNING_PASSWORD: ${{ secrets.JAVA_SIGNING_PASSWORD }} + JAVAVERSION: "11" jobs: build_sdk: name: build_sdk @@ -51,6 +57,16 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{matrix.pythonversion}} + - name: Setup Java + uses: actions/setup-java@v3 + with: + java-version: ${{ env.JAVAVERSION }} + distribution: temurin + cache: gradle + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + gradle-version: "7.6" - name: Download provider + tfgen binaries uses: actions/download-artifact@v3 with: @@ -88,6 +104,7 @@ jobs: - python - dotnet - go + - java nodeversion: - 14.x pythonversion: @@ -270,6 +287,16 @@ jobs: - 14.x pythonversion: - "3.7" + publish_java_sdk: + runs-on: ubuntu-latest + continue-on-error: true + needs: publish + name: publish_java_sdk + steps: + - name: Publish Java SDK + uses: pulumi/pulumi-package-publisher@0058a106b68d8277f17bbea0cd29b2ff6e671adc + with: + sdk: java test: name: test needs: build_sdk @@ -348,6 +375,7 @@ jobs: - python - dotnet - go + # TODO: Add java here. But it needs to have tests to run. nodeversion: - 14.x pythonversion: