Upload Windows binaries earlier

.github/workflows/deployment-arm64.yml

@@ -172,11 +172,6 @@ jobs:
         if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
         shell: bash
         run: |
-          mkdir ${{runner.temp}}/sshkey
-          if [[ -z "${{ secrets.buildJabRefPrivateKey }}" ]]; then
-            echo "buildJabRefPrivateKey is empty, exiting"
-            exit 1
-          fi
           rsync -Pavz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" -e 'ssh -p 9922 -i ~/.ssh/id_rsa' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/
       - name: Upload to GitHub workflow artifacts store
         if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}

.github/workflows/deployment.yml

@@ -35,6 +35,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  rsynctest:
+    runs-on: windows-latest
+    steps:
+      - name: Setup rsync
+        uses: GuillaumeFalourd/setup-rsync@test
+      - name: Setup ssh key
+        shell: bash
+        run: |
+          echo "${{ secrets.buildJabRefPrivateKey }}" > ~/sshkey
+          chmod 600 ~/sshkey
+          mkdir -p build/distribution/
+          touch build/distribution/test.txt
+      - name: Upload to builds.jabref.org (Windows)
+        shell: bash
+        run: |
+          rsync -Pavz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /tmp/test && rsync" -e 'ssh -p 9922 -i ~/sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/tmp/test/
   build:
     strategy:
       fail-fast: false
@@ -53,7 +69,11 @@ jobs:
             displayName: macOS
             eaJDK: https://files.jabref.org/jdks/jdk-macos-x64.tar.gz
     runs-on: ${{ matrix.os }}
-    name: Create installer and portable version for ${{ matrix.displayName }}
+    outputs:
+      major: ${{ steps.gitversion.outputs.Major }}
+      minor: ${{ steps.gitversion.outputs.Minor }}
+      branchname: ${{ steps.gitversion.outputs.branchName }}
+    name: ${{ matrix.displayName }} installer and portable version
     steps:
       - name: Check secrets presence
         id: checksecrets
@@ -105,7 +125,7 @@ jobs:
          cat gradle.properties
 
          sed -i "s/JavaLanguageVersion.of(20)/JavaLanguageVersion.of(21)/" build.gradle
-      - name: Setup JDK jabref-fix (ubuntu)
+      - name: Setup JDK jabref-fix (linux)
         if: (matrix.os == 'ubuntu-latest')
         shell: bash
         run: |
@@ -134,31 +154,24 @@ jobs:
           cat gradle.properties
 
           sed -i'.bak' -e "s/JavaLanguageVersion.of(20)/JavaLanguageVersion.of(21)/" build.gradle
-      - name: Setup OSX key chain (macOS)
+      - name: Setup macOS key chain
         if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES')
         uses: apple-actions/import-codesign-certs@v2
         with:
           p12-file-base64: ${{ secrets.OSX_SIGNING_CERT }}
           p12-password: ${{ secrets.OSX_CERT_PWD }}
           keychain-password: jabref
-      - name: Setup OSX key chain on OSX for app id cert (macOS)
+      - name: Setup macOS key chain for app id cert
         if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES')
         uses: apple-actions/import-codesign-certs@v2
         with:
           p12-file-base64: ${{ secrets.OSX_SIGNING_CERT_APPLICATION }}
           p12-password: ${{ secrets.OSX_CERT_PWD }}
           create-keychain: false
           keychain-password: jabref
-      - name: Build runtime image (non-macOS)
-        if: (matrix.os != 'macos-latest')
-        run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" jlinkZip
-      - name: Prepare merged jars and modules dir (macos)
+      - name: Prepare merged jars and modules dir (macOS)
         if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES')
         run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" prepareModulesDir
-      - name: Build installer (non-macos)
-        if: (matrix.os != 'macos-latest')
-        shell: bash
-        run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" jpackage
       - name: Build dmg (macOS)
         if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES')
         shell: bash
@@ -209,7 +222,11 @@ jobs:
           --resource-dir buildres/mac \
           --file-associations buildres/mac/bibtexAssociations.properties \
           --jlink-options --bind-services
-      - name: Package application image (non-macOS)
+      - name: Build runtime image and installer (linux, Windows)
+        if: (matrix.os != 'macos-latest')
+        shell: bash
+        run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" jpackage jlinkZip
+      - name: Package application image (linux, Windows)
         if: (matrix.os != 'macos-latest')
         shell: bash
         run: ${{ matrix.archivePortable }}
@@ -230,35 +247,50 @@ jobs:
           ar -m -c -a sdsd jabref_${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}_amd64_repackaged.deb debian-binary control.tar.xz data.tar.xz
           rm debian-binary control.tar.* data.tar.*
           mv -f jabref_${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}_amd64_repackaged.deb  jabref_${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}_amd64.deb
-      - name: Upload to builds.jabref.org (ubuntu)
-        if: (matrix.os == 'ubuntu-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue'))
-        uses: Pendect/action-rsyncer@v2.0.0
-        env:
-          DEPLOY_KEY: ${{ secrets.buildJabRefPrivateKey }}
-        with:
-          flags: -vaz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync"
-          options: ''
-          ssh_options: '-p 9922'
-          src: 'build/distribution/'
-          dest: jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/
-      - name: Upload to GitHub workflow artifacts store (Windows)
-        if: (matrix.os == 'windows-latest') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue'))
-        uses: actions/upload-artifact@v3
-        with:
-          name: JabRef-${{ matrix.displayName }}
-          path: build/distribution
+      - name: Setup rsync (macOS)
+        if: matrix.os == 'macos-latest'
+        run: brew install rsync
+      - name: Setup rsync (Windows)
+        if: matrix.os == 'windows-latest'
+        # We want to have rsync available at this place to avoid uploading and downloading from GitHub artifact store (taking > 5 minutes in total)
+        # We cannot use "action-rsyncer", because that requires Docker which is unavailable on Windows
+        # We cannot use "setup-rsync", because that does not work on Windows
+        # We do not use egor-tensin/setup-cygwin@v4, because it replaces the default shell
+        run: choco install --no-progress rsync
+      - name: Setup ssh key
+        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue'))
+        run: |
+          echo "${{ secrets.buildJabRefPrivateKey }}" > sshkey
+          chmod 600 sshkey
+      - name: Upload to builds.jabref.org (Windows)
+        if: (matrix.os == 'windows-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue'))
+        shell: cmd
+        # for rsync installed by chocolatey, we need the ssh.exe delivered with that installation
+        run: |
+          rsync -Pavz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" -e 'C:\ProgramData\chocolatey\lib\rsync\tools\bin\ssh.exe -p 9922 -i sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/
+      - name: Upload to builds.jabref.org (linux, macOS)
+        # macOS: Negated condition of "Upload to GitHub workflow artifacts store (macOS)"
+        #        Reason: We either upload the non-notarized files - or notarize the files later (and upload these later)
+        if: >
+          ${{
+            (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) && (steps.checksecrets.outputs.secretspresent == 'YES') &&
+            ((matrix.os == 'ubuntu-latest') ||
+             (matrix.os == 'macos-latest') && !((startsWith(github.ref, 'refs/tags/') || inputs.notarization == true))) }}
+        shell: bash
+        run: |
+          rsync -Pavz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" -e 'ssh -p 9922 -i sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/
       - name: Upload to GitHub workflow artifacts store (macOS)
-        if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue'))
+        if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (startsWith(github.ref, 'refs/tags/') || inputs.notarization == true)
         uses: actions/upload-artifact@v3
         with:
           # tbn = to-be-notarized
           name: JabRef-macOS-tbn
           path: build/distribution
-  notarize: # outsourced in a separate job to be able to rerun if this fails for timeouts
-    name: Notarize and package macOS binaries
-    runs-on: macos-latest
+  announce:
+    name: Comment on Pull Request
+    runs-on: ubuntu-latest
     needs: [build]
-    if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
+    if: ${{ github.event_name == 'pull_request' }}
     steps:
       - name: Check secrets presence
         id: checksecrets
@@ -271,59 +303,19 @@ jobs:
           fi
         env:
           BUILDJABREFPRIVATEKEY: ${{ secrets.buildJabRefPrivateKey }}
-      - name: Fetch all history for all tags and branches
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: 'true'
-          show-progress: 'false'
-      - name: Install GitVersion
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: gittools/actions/gitversion/setup@v0.10.2
-        with:
-          versionSpec: "5.x"
-      - name: Run GitVersion
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        id: gitversion
-        uses: gittools/actions/gitversion/execute@v0.10.2
-      - name: Get macOS binaries
+      - name: Comment PR
         if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: actions/download-artifact@master
-        with:
-          name: JabRef-macOS-tbn
-          path: build/distribution/
-      - name: Notarize dmg
-        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (startsWith(github.ref, 'refs/tags/') ||  inputs.notarization == true)
-        shell: bash
-        run: |
-          xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}"
-          xcrun notarytool submit build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.dmg --keychain-profile "notarytool-profile" --wait
-          xcrun stapler staple build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.dmg
-      - name: Notarize pkg
-        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (startsWith(github.ref, 'refs/tags/') ||  inputs.notarization == true)
-        shell: bash
-        run: |
-          xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}"
-          xcrun notarytool submit build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.pkg --keychain-profile "notarytool-profile" --wait
-          xcrun stapler staple build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.pkg
-      - name: Package application image
-        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (matrix.os != 'macos-latest')
-        shell: bash
-        run: ${{ matrix.archivePortable }}
-      - name: Upload to GitHub workflow artifacts store
-        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue'))
-        uses: actions/upload-artifact@v3
+        uses: thollander/actions-comment-pull-request@v2
         with:
-          name: JabRef-macOS
-          path: build/distribution
-  upload:
-    strategy:
-      fail-fast: false
-    name: Upload binaries on builds.jabref.org
-    runs-on: ubuntu-latest
-    needs: [build, notarize]
-    if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
+          message: |
+            The build of this PR is available at <https://builds.jabref.org/pull/${{ github.event.pull_request.number }}/merge>.
+          comment_tag: download-link
+          mode: recreate
+  notarize: # outsourced in a separate job to be able to rerun if this fails for timeouts
+    name: macOS notarization
+    runs-on: macos-latest
+    needs: [build]
+    if: ${{ startsWith(github.ref, 'refs/tags/') || inputs.notarization == true }}
     steps:
       - name: Check secrets presence
         id: checksecrets
@@ -336,58 +328,30 @@ jobs:
           fi
         env:
           BUILDJABREFPRIVATEKEY: ${{ secrets.buildJabRefPrivateKey }}
-      - name: Checkout source
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: 'true'
-          show-progress: 'false'
-      - name: Install GitVersion
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: gittools/actions/gitversion/setup@v0.10.2
-        with:
-          versionSpec: '5.x'
-      - name: Run GitVersion
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        id: gitversion
-        uses: gittools/actions/gitversion/execute@v0.10.2
-      - name: Get Windows binaries
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: actions/download-artifact@master
-        with:
-          name: JabRef-windows
-          path: build/distribution
-      - name: Get macOS binaries unsigned
-        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (inputs.notarization == false && !startsWith(github.ref, 'refs/tags/'))
+      - name: Download from GitHub workflow artifacts store (macOS)
+        if: (steps.checksecrets.outputs.secretspresent == 'YES')
         uses: actions/download-artifact@master
         with:
           name: JabRef-macOS-tbn
           path: build/distribution/
-      - name: Get macOS binaries notarized
-        if: (steps.checksecrets.outputs.secretspresent == 'YES') && (inputs.notarization == true || startsWith(github.ref, 'refs/tags/'))
-        uses: actions/download-artifact@master
-        with:
-          name: JabRef-macOS
-          path: build/distribution/
-      # Upload to build server using rsync
-      # The action runs on linux only (because it is a Dockerized action), therefore it is embedded in a separate workflow
+      - name: Notarize dmg
+        if: (steps.checksecrets.outputs.secretspresent == 'YES')
+        shell: bash
+        run: |
+          xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}"
+          xcrun notarytool submit build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.dmg --keychain-profile "notarytool-profile" --wait
+          xcrun stapler staple build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.dmg
+      - name: Notarize pkg
+        if: (steps.checksecrets.outputs.secretspresent == 'YES')
+        shell: bash
+        run: |
+          xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}"
+          xcrun notarytool submit build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.pkg --keychain-profile "notarytool-profile" --wait
+          xcrun stapler staple build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.pkg
       - name: Upload to builds.jabref.org
-        if: steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: Pendect/action-rsyncer@v2.0.0
-        env:
-          DEPLOY_KEY: ${{ secrets.buildJabRefPrivateKey }}
-        with:
-          flags: -vaz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync"
-          options: ''
-          ssh_options: '-p 9922'
-          src: 'build/distribution/'
-          dest: jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/
-      - name: Comment PR
-        if: github.event_name == 'pull_request' && steps.checksecrets.outputs.secretspresent == 'YES'
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          message: |
-            The build of this PR is available at <https://builds.jabref.org/pull/${{ github.event.pull_request.number }}/merge>.
-          comment_tag: download-link
-          mode: recreate
+        if: (steps.checksecrets.outputs.secretspresent == 'YES')
+        shell: bash
+        run: |
+          echo "${{ secrets.buildJabRefPrivateKey }}" > sshkey
+          chmod 600 sshkey
+          rsync -Pavz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ needs.build.outputs.branchname }} && rsync" -e 'ssh -p 9922 -i sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ needs.build.outputs.branchname }}/

.github/workflows/refresh-journal-lists.yml

@@ -41,7 +41,7 @@ jobs:
           cache: 'gradle'
       - name: Check whether journal-list.mv can be generated (the "real" generation is done inside JabRef's build process)
         run: |
-          ./gradlew generateJournalAbbreviationList
+          ./gradlew generateJournalListMV
       - uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.GITHUB_TOKEN }}