From 73a8134c90134167e71252d86a7c1344291ebfa0 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 17 Jun 2022 19:44:39 +0000 Subject: [PATCH] Rework doctesting and docs uploading I want to minimize the number of times we actually build Julia on CI. This PR re-uses the output of `build_x86_64-linux-gnu`, and also adds a second, signed step to deploy the docs using an encrypted `DOCUMENTER_KEY`. Use `aws_uploader` --- .../main/launch_signed_jobs.yml.signature | 4 +- pipelines/main/launch_upload_jobs.yml | 3 + .../main/launch_upload_jobs.yml.signature | Bin 96 -> 96 bytes pipelines/main/misc/deploy_docs.yml | 81 ++++++++++++++++++ pipelines/main/misc/doctest.yml | 32 +++++-- pipelines/main/platforms/upload_linux.yml | 4 +- .../scheduled/coverage/coverage.yml.signature | 2 +- .../launch_signed_jobs.yml.signature | 2 +- utilities/deploy_docs.sh | 13 +++ 9 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 pipelines/main/misc/deploy_docs.yml create mode 100644 utilities/deploy_docs.sh diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature index 31fb2469..1f6e4053 100644 --- a/pipelines/main/launch_signed_jobs.yml.signature +++ b/pipelines/main/launch_signed_jobs.yml.signature @@ -1,3 +1 @@ -Salted__bu -^A+ y%bv(e rRNy;G Qm -Q? 0\Q?1tDU \ No newline at end of file +Salted__&10е|w4$u!v#CʽO=,¦_Z/UVowc)ӕ4,tMxV),x-^ԋ~ Fy? \ No newline at end of file diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 2916f690..2389b728 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -22,6 +22,9 @@ steps: .buildkite/pipelines/main/platforms/upload_macos.arches \ .buildkite/pipelines/main/platforms/upload_macos.yml + # Launch doctest deploy job + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml + # Don't share this with buildkite's env display unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET agents: diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature index 8e7990fec2a5faee20244c194b435345af6b5bcd..86e26c7f6d13b4a05ef18f2b604209bbe72d7b1c 100644 GIT binary patch literal 96 zcmV-m0H6O;VQh3|WM5x-yQLCQm198D^Ccc7oFR|Y CtuIvo literal 96 zcmV-m0H6O;VQh3|WM5y(DB+x}O@wz0sJ`bVpHN|&Pb{Bg(`5(Jjkn~3MV^_nx7F$* zavsot6k5KlD|6w?d> /home/juliaci/.ssh/known_hosts 2>/dev/null + git clone git@github.com:JuliaLang/docs.julialang.org -b gh-pages repo + cd repo + + echo "--- deploy new docs" + # Delete everything except for `.git` + shopt -s extglob + rm -rf !(".git") + shopt -u extglob + + # Extract new docs build + tar -zxf ../julia-*-htmldocs.tar.gz + + # Build commit message + echo "build based on $${BUILDKITE_COMMIT}\n" > ../commit_message + echo "build link: $${BUILDKITE_BUILD_URL}" >> ../commit_message + + # Add all files to our new commit + git config --global user.name "Documenter.jl" + git config --global user.email "documenter@juliadocs.github.io" + git add . + git commit -a -F ../commit_message + + if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then + # If we're testing new CI configurations, push to a temporary branch, then delete it + git push origin gh-pages:gh-pages-buildkite-$${BUILDKITE_COMMIT} + git push -D origin gh-pages-buildkite-$${BUILDKITE_COMMIT} + else + git push origin gh-pages:gh-pages + fi + timeout_in_minutes: 45 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} + diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml index c8dbc430..7e86dd29 100644 --- a/pipelines/main/misc/doctest.yml +++ b/pipelines/main/misc/doctest.yml @@ -3,6 +3,8 @@ steps: steps: - label: "doctest" key: doctest + depends_on: + - "build_x86_64-linux-gnu" plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -10,27 +12,39 @@ steps: - JuliaCI/julia#v1: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled - version: '1.6' + version: '1' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.11/package_linux.x86_64.tar.gz + rootfs_treehash: "3c62a3dd4116a1b698c103f1e090b824e4cc9ff5" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" commands: | - echo "--- Build Julia from source" - make --output-sync -j 6 + # Download pre-built julia, extract into `usr/` + set -x + buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-linux-x86_64.tar.gz + ln -s ./usr/bin/julia ./julia echo "--- Print Julia version info" + export JULIA_NUM_THREADS=1 ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "--- Build Julia docs" - make docs + TARBALL_NAME="$$(echo julia-*-linux-x86_64.tar.gz)" + export DOCUMENTER_ARCHIVE="$${TARBALL_NAME%-linux-x86_64.tar.gz}-htmldocs.tar.gz" - echo "--- Run Julia doctests" - JULIA_NUM_THREADS=1 make -C doc doctest=true + echo "--- Run Julia doctests/build HTML docs" + cd doc + make deploy doctest=true + + echo "--- Compress and upload HTML docs" + buildkite-agent artifact upload "$${DOCUMENTER_ARCHIVE}" + env: + # Give a fake documenter key to get Documenter to try and deploy for us + DOCUMENTER_KEY: "" timeout_in_minutes: 45 agents: queue: "julia" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml index 9b0558dc..3d790aca 100644 --- a/pipelines/main/platforms/upload_linux.yml +++ b/pipelines/main/platforms/upload_linux.yml @@ -27,8 +27,8 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.7/aws_uploader.x86_64.tar.gz - rootfs_treehash: "0d5ad94ba902eea4d9a7d220b49c144626a102ea" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.22/aws_uploader.x86_64.tar.gz + rootfs_treehash: "486b8618fd0184dd869c9282a73bb08aa1411e82" uid: 1000 gid: 1000 - staticfloat/cryptic#v2: diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature index b0104a98..1c1a4008 100644 --- a/pipelines/scheduled/coverage/coverage.yml.signature +++ b/pipelines/scheduled/coverage/coverage.yml.signature @@ -1 +1 @@ -Salted__ڰ]K:m! "v0$0$Z'Zᣳ>}mf SsghE賬V2rQ \ No newline at end of file +Salted__G.;8zX/;I /dev/null && pwd )" + +echo "--- Download built docs" +buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" + +echo "--- Deploy docs" +export DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)" +echo "Do something here!"