Rework doctesting and docs uploading

I want to minimize the number of times we actually build Julia on CI. This PR re-uses the output of `build_x86_64-linux-gnu`, and also adds a second, signed step to deploy the docs using an encrypted `DOCUMENTER_KEY`. Use `aws_uploader`
JuliaCI · Jul 12, 2022 · 685149c · 685149c
1 parent 74f226e
commit 685149c
Show file tree

Hide file tree

Showing 9 changed files with 123 additions and 18 deletions.
diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature
@@ -1,3 +1 @@
-Salted__��b��u���
-^A��+��y%b���v(�e�rR��N��y;�G� Q������m�
-��Q?����0�\��Q?1�tDU��
+Salted__JLkN�A�ی�%4k4�Xx��\����7CV	A�K����ˮ��9�a~X�O�-ԗ��Q_o�u�;S׶;��#"�Uq��Y�
diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml
@@ -22,6 +22,9 @@ steps:
               .buildkite/pipelines/main/platforms/upload_macos.arches \
               .buildkite/pipelines/main/platforms/upload_macos.yml
 
+          # Launch doctest deploy job
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml
+
           # Don't share this with buildkite's env display
           unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
         agents:

diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature
@@ -1,2 +1,2 @@
-Salted__�(ᜬM�w���%�Pa�O,�d�eҍ��E������"rЀZ��+s��|�
-��g��I��خŖ��>���ڪ=x����Z�/�
+Salted__���L�ӡ+�R5N��TgZPB�*>��:P� O�i5Ρ��b�8
+±Lu�V�%魱З����~��������˃���k<%�

diff --git a/pipelines/main/misc/deploy_docs.yml b/pipelines/main/misc/deploy_docs.yml
@@ -0,0 +1,77 @@
+steps:
+  - group: "Upload"
+    steps:
+      - label: "upload docs"
+        depends_on:
+          - "doctest"
+        # We only upload if one of the following criteria are true:
+        # 1. The branch is `master`.
+        # 2. The branch is `release-*`.
+        # 3. The build is a tag build AND the tag is `v*`.
+        # 4. The pipeline is `julia-buildkite`.
+        if: ((build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/)) || (pipeline.slug == "julia-buildkite")
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: "./.buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/julia#v1:
+              # Drop default "registries" directory, so it is not persisted from execution to execution
+              persist_depot_dirs: packages,artifacts,compiled
+              # Use Julia v1.7 for upload, that's the version embedded in the documenter manifest
+              version: '1.7'
+          - staticfloat/sandbox#v1:
+              # Use `aws_uploader` image, since it has things like `git` and `ssh` and whatnot
+              rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v5.27/aws_uploader.x86_64.tar.gz"
+              rootfs_treehash: "cd5a62e570bda70932217641ec715b2f9fc1cd82"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # Include `/cache/repos` so that our `git` version introspection works.
+                - "/cache/repos:/cache/repos"
+          - staticfloat/cryptic#v2:
+              files:
+                - .buildkite/secrets/ssh_docs_deploy
+          - staticfloat/ssh-agent#v1:
+              keyfiles:
+                - .buildkite/secrets/ssh_docs_deploy
+        commands: |
+          buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" .
+
+          echo "--- clone docs.julialang.org"
+          mkdir -p ~/.ssh
+          ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
+          git clone git@github.com:JuliaLang/docs.julialang.org -b gh-pages repo
+          cd repo
+
+          echo "--- deploy new docs"
+          # Delete everything except for `.git`
+          shopt -s extglob
+          rm -rf !(".git")
+          shopt -u extglob
+
+          # Extract new docs build
+          tar -zxf ../julia-*-htmldocs.tar.gz
+
+          # Build commit message
+          echo "build based on $${BUILDKITE_COMMIT}\n" > ../commit_message
+          echo "build link: $${BUILDKITE_BUILD_URL}" >> ../commit_message
+
+          # Add all files to our new commit
+          git config --global user.name "Documenter.jl"
+          git config --global user.email "documenter@juliadocs.github.io"
+          git add .
+          git commit -a -F ../commit_message
+
+          if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then
+            # If we're testing new CI configurations, push to a temporary branch, then delete it
+            git push origin gh-pages:gh-pages-buildkite-$${BUILDKITE_COMMIT}
+            git push -D origin gh-pages-buildkite-$${BUILDKITE_COMMIT}
+          else
+            git push origin gh-pages:gh-pages
+          fi
+        timeout_in_minutes: 45
+        agents:
+          queue: "julia"
+          sandbox_capable: "true"
+          os: "linux"
+          arch: "x86_64"
diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml
@@ -3,34 +3,48 @@ steps:
     steps:
       - label: "doctest"
         key: doctest
+        depends_on:
+          - "build_x86_64-linux-gnu"
         plugins:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
           - JuliaCI/julia#v1:
               # Drop default "registries" directory, so it is not persisted from execution to execution
               persist_depot_dirs: packages,artifacts,compiled
-              version: '1.6'
+              version: '1'
           - staticfloat/sandbox#v1:
-              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
-              rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.11/package_linux.x86_64.tar.gz
+              rootfs_treehash: "3c62a3dd4116a1b698c103f1e090b824e4cc9ff5"
               uid: 1000
               gid: 1000
               workspaces:
                 # Include `/cache/repos` so that our `git` version introspection works.
                 - "/cache/repos:/cache/repos"
         commands: |
-          echo "--- Build Julia from source"
-          make --output-sync -j 6
+          # Download pre-built julia, extract into `usr/`
+          set -x
+          buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' .
+          mkdir -p ./usr
+          tar -C ./usr --strip-components=1 -zxf julia-*-linux-x86_64.tar.gz
+          ln -s ./usr/bin/julia ./julia
 
           echo "--- Print Julia version info"
+          export JULIA_NUM_THREADS=1
           ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
 
-          echo "--- Build Julia docs"
-          make docs
+          TARBALL_NAME="$$(echo julia-*-linux-x86_64.tar.gz)"
+          export DOCUMENTER_ARCHIVE="$${TARBALL_NAME%-linux-x86_64.tar.gz}-htmldocs.tar.gz"
 
-          echo "--- Run Julia doctests"
-          JULIA_NUM_THREADS=1 make -C doc doctest=true
+          echo "--- Run Julia doctests/build HTML docs"
+          cd doc
+          make deploy doctest=true
+
+          echo "--- Compress and upload HTML docs"
+          buildkite-agent artifact upload "$${DOCUMENTER_ARCHIVE}"
+        env:
+          # Give a fake documenter key to get Documenter to try and deploy for us
+          DOCUMENTER_KEY: ""
         timeout_in_minutes: 45
         agents:
           queue: "julia"

diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml
@@ -27,8 +27,8 @@ steps:
               persist_depot_dirs: packages,artifacts,compiled
               version: '1'
           - staticfloat/sandbox#v1:
-              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.7/aws_uploader.x86_64.tar.gz
-              rootfs_treehash: "0d5ad94ba902eea4d9a7d220b49c144626a102ea"
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.22/aws_uploader.x86_64.tar.gz
+              rootfs_treehash: "486b8618fd0184dd869c9282a73bb08aa1411e82"
               uid: 1000
               gid: 1000
           - staticfloat/cryptic#v2:

diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature
@@ -1 +1 @@
-Salted__ڰ�]����K��:�m���!��"�v0$�0�������$Z��'�����Z�ᣳ>}��m���fSsghE�賬���V�2r�Q
+Salted__�osd���t(�B>�	�nv�,��� ߺ����3lY��8��LU�_��8%�n��3O�(|A�me����0�b�EY�E��ݶb
diff --git a/pipelines/scheduled/launch_signed_jobs.yml.signature b/pipelines/scheduled/launch_signed_jobs.yml.signature
@@ -1 +1 @@
-Salted__%�6���kc����;�t���X��<���o�F̛M@�lrT� ZT�:-u=U�X��-D�%������ʌ���0������B��CNO�F
+Salted__$WE�&�?�L�m˜����yB������o[R֐p�	�k�G+����W�D��;}���z���s�uJ�3�&lM�%q�a��\W_
diff --git a/utilities/deploy_docs.sh b/utilities/deploy_docs.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Get the `.buildkite/utilities/docs` folder path
+DOCS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+echo "--- Download built docs"
+buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz"
+
+echo "--- Deploy docs"
+export DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)"
+echo "Do something here!"