diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature index 31fb2469..1f6e4053 100644 --- a/pipelines/main/launch_signed_jobs.yml.signature +++ b/pipelines/main/launch_signed_jobs.yml.signature @@ -1,3 +1 @@ -Salted__bu -^A+ y%bv(e rRNy;G Qm -Q? 0\Q?1tDU \ No newline at end of file +Salted__&10е|w4$u!v#CʽO=,¦_Z/UVowc)ӕ4,tMxV),x-^ԋ~ Fy? \ No newline at end of file diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 2916f690..2389b728 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -22,6 +22,9 @@ steps: .buildkite/pipelines/main/platforms/upload_macos.arches \ .buildkite/pipelines/main/platforms/upload_macos.yml + # Launch doctest deploy job + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml + # Don't share this with buildkite's env display unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET agents: diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature index 8e7990fe..86e26c7f 100644 Binary files a/pipelines/main/launch_upload_jobs.yml.signature and b/pipelines/main/launch_upload_jobs.yml.signature differ diff --git a/pipelines/main/misc/deploy_docs.yml b/pipelines/main/misc/deploy_docs.yml new file mode 100644 index 00000000..c65fad29 --- /dev/null +++ b/pipelines/main/misc/deploy_docs.yml @@ -0,0 +1,81 @@ +steps: + - group: "Upload" + steps: + - label: "upload docs" + depends_on: + - "doctest" + # We only upload if one of the following criteria are true: + # 1. The branch is `master`. + # 2. The branch is `release-*`. + # 3. The build is a tag build AND the tag is `v*`. + # 4. The pipeline is `julia-buildkite`. + if: ((build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/)) || (pipeline.slug == "julia-buildkite") + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + # Use Julia v1.7 for upload, that's the version embedded in the documenter manifest + version: '1.7' + - staticfloat/sandbox#v1: + # Use `aws_uploader` image, since it has things like `git` and `ssh` and whatnot + rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v5.27/aws_uploader.x86_64.tar.gz" + rootfs_treehash: "cd5a62e570bda70932217641ec715b2f9fc1cd82" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + - staticfloat/cryptic#v2: + files: + - .buildkite/secrets/ssh_docs_deploy + - staticfloat/ssh-agent#v1: + keyfiles: + - .buildkite/secrets/ssh_docs_deploy + commands: | + buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" . + + echo "--- clone docs.julialang.org" + mkdir -p /home/juliaci/.ssh + ssh-keyscan github.com >> /home/juliaci/.ssh/known_hosts 2>/dev/null + git clone git@github.com:JuliaLang/docs.julialang.org -b gh-pages repo + cd repo + + echo "--- deploy new docs" + # Delete everything except for `.git` + shopt -s extglob + rm -rf !(".git") + shopt -u extglob + + # Extract new docs build + tar -zxf ../julia-*-htmldocs.tar.gz + + # Build commit message + echo "build based on $${BUILDKITE_COMMIT}\n" > ../commit_message + echo "build link: $${BUILDKITE_BUILD_URL}" >> ../commit_message + + # Add all files to our new commit + git config --global user.name "Documenter.jl" + git config --global user.email "documenter@juliadocs.github.io" + git add . + git commit -a -F ../commit_message + + if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then + # If we're testing new CI configurations, push to a temporary branch, then delete it + git push origin gh-pages:gh-pages-buildkite-$${BUILDKITE_COMMIT} + git push -D origin gh-pages-buildkite-$${BUILDKITE_COMMIT} + else + git push origin gh-pages:gh-pages + fi + timeout_in_minutes: 45 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} + diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml index c8dbc430..7e86dd29 100644 --- a/pipelines/main/misc/doctest.yml +++ b/pipelines/main/misc/doctest.yml @@ -3,6 +3,8 @@ steps: steps: - label: "doctest" key: doctest + depends_on: + - "build_x86_64-linux-gnu" plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -10,27 +12,39 @@ steps: - JuliaCI/julia#v1: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled - version: '1.6' + version: '1' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.11/package_linux.x86_64.tar.gz + rootfs_treehash: "3c62a3dd4116a1b698c103f1e090b824e4cc9ff5" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" commands: | - echo "--- Build Julia from source" - make --output-sync -j 6 + # Download pre-built julia, extract into `usr/` + set -x + buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-linux-x86_64.tar.gz + ln -s ./usr/bin/julia ./julia echo "--- Print Julia version info" + export JULIA_NUM_THREADS=1 ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "--- Build Julia docs" - make docs + TARBALL_NAME="$$(echo julia-*-linux-x86_64.tar.gz)" + export DOCUMENTER_ARCHIVE="$${TARBALL_NAME%-linux-x86_64.tar.gz}-htmldocs.tar.gz" - echo "--- Run Julia doctests" - JULIA_NUM_THREADS=1 make -C doc doctest=true + echo "--- Run Julia doctests/build HTML docs" + cd doc + make deploy doctest=true + + echo "--- Compress and upload HTML docs" + buildkite-agent artifact upload "$${DOCUMENTER_ARCHIVE}" + env: + # Give a fake documenter key to get Documenter to try and deploy for us + DOCUMENTER_KEY: "" timeout_in_minutes: 45 agents: queue: "julia" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml index 9b0558dc..3d790aca 100644 --- a/pipelines/main/platforms/upload_linux.yml +++ b/pipelines/main/platforms/upload_linux.yml @@ -27,8 +27,8 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.7/aws_uploader.x86_64.tar.gz - rootfs_treehash: "0d5ad94ba902eea4d9a7d220b49c144626a102ea" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.22/aws_uploader.x86_64.tar.gz + rootfs_treehash: "486b8618fd0184dd869c9282a73bb08aa1411e82" uid: 1000 gid: 1000 - staticfloat/cryptic#v2: diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature index b0104a98..1c1a4008 100644 --- a/pipelines/scheduled/coverage/coverage.yml.signature +++ b/pipelines/scheduled/coverage/coverage.yml.signature @@ -1 +1 @@ -Salted__ڰ]K:m! "v0$0$Z'Zᣳ>}mf SsghE賬V2rQ \ No newline at end of file +Salted__G.;8zX/;I /dev/null && pwd )" + +echo "--- Download built docs" +buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" + +echo "--- Deploy docs" +export DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)" +echo "Do something here!"