From 0dfcefddd4e12adaaf298a350a4374bacae2f6f6 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 4 Oct 2021 08:25:50 -0400 Subject: [PATCH] CI (Buildkite): add `package_musl64` and `tester_musl64` (Linux, musl libc, 64-bit, x86_64 architecture) (#42476) (cherry picked from commit 1195f5adf05d78a4001b981bedd4b68a865cf8c3) --- .../main/launch_unsigned_builders.yml | 14 +++++++----- .../main/platforms/package_linux.arches | 13 ++++++----- .../main/platforms/package_linux.yml | 17 +++++++++----- .../pipelines/main/platforms/platforms.sh | 22 ++++++++++--------- .../main/platforms/tester_linux.arches | 17 +++++++------- .../pipelines/main/platforms/tester_linux.yml | 12 +++++----- 6 files changed, 54 insertions(+), 41 deletions(-) diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml index d430ad1503ff9..4ff6e310e3c12 100644 --- a/.buildkite/pipelines/main/launch_unsigned_builders.yml +++ b/.buildkite/pipelines/main/launch_unsigned_builders.yml @@ -13,19 +13,21 @@ # something about the privileged steps. steps: - - label: ":buildkite: Launch unsigned builders" + - label: ":buildkite: Launch unsigned jobs" commands: | - # Launch the `whitespace` builder first, because we want that builder to finish as quickly as possible. - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml - - # Launch the miscellaneous builders in alphabetical order. + # Launch the miscellaneous jobs in alphabetical order. buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml - # Launch all of the platform builders. + # Launch all of the platform jobs. bash .buildkite/pipelines/main/platforms/platforms.sh package_linux bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux + + # Launch the `whitespace` job last. Uploading it last actually causes it to start + # first. We want this job to start first because we want it to finish as quickly + # as possible. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml agents: queue: julia diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches index d83bd798242ac..58d5b87f4c3a3 100644 --- a/.buildkite/pipelines/main/platforms/package_linux.arches +++ b/.buildkite/pipelines/main/platforms/package_linux.arches @@ -1,6 +1,7 @@ -# ARCH ARCH_LABEL ROOTFS_ARCH TIMEOUT ROOTFS_TAG ROOTFS_TREE -# aarch64 aarch64 aarch64 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff -# armv7l armv7l armv7l 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 -32 32 i686 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 -# ppc64le ppc64le powerpc64le 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 -64 64 x86_64 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +# PLATFORM ARCH ARCH_LABEL ROOTFS_ARCH ALLOW_FAIL TIMEOUT ROOTFS_TAG ROOTFS_TREE +# linux aarch64 aarch64 aarch64 false 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff +# linux armv7l armv7l armv7l false 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 +linux 32 32 i686 false 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 +# linux ppc64le ppc64le powerpc64le false 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 +linux 64 64 x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +musl 64 64 x86_64 false 60 v3.19 e6a2730e37c386c46915b2650d6aaaa398195152 diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml index f6f937738e431..9108c5a41a212 100644 --- a/.buildkite/pipelines/main/platforms/package_linux.yml +++ b/.buildkite/pipelines/main/platforms/package_linux.yml @@ -4,15 +4,15 @@ agents: sandbox.jl: "true" os: "linux" steps: - - label: "package_linux${ARCH_LABEL?}" - key: package_linux${ARCH_LABEL?} + - label: "package_${PLATFORM?}${ARCH_LABEL?}" + key: package_${PLATFORM?}${ARCH_LABEL?} plugins: - JuliaCI/julia#v1: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz rootfs_treehash: "${ROOTFS_TREE?}" uid: 1000 gid: 1000 @@ -24,9 +24,11 @@ steps: echo "--- Print the full and short commit hashes" SHORT_COMMIT_LENGTH=10 SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-` ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}" + ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" + JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-` + JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}" + echo "The full commit is: $${BUILDKITE_COMMIT:?}" echo "The short commit is: $${SHORT_COMMIT:?}" echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" @@ -43,7 +45,10 @@ steps: echo "--- Create build artifacts" make -j 8 binary-dist + ls -l $${JULIA_BINARYDIST:?} + if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then + mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?} + fi ls -l $${ARTIFACT_FILENAME:?} - echo "--- Upload build artifacts" buildkite-agent artifact upload $${ARTIFACT_FILENAME:?} diff --git a/.buildkite/pipelines/main/platforms/platforms.sh b/.buildkite/pipelines/main/platforms/platforms.sh index d58daeb2a9929..760d65c4e842f 100755 --- a/.buildkite/pipelines/main/platforms/platforms.sh +++ b/.buildkite/pipelines/main/platforms/platforms.sh @@ -2,9 +2,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -PLATFORM="$1" +OS="$1" -cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read _line; do +cat "$SCRIPT_DIR/$OS.arches" | tr -s ' ' | while read _line; do # Remove whitespace from the beginning and end of each line line=`echo $_line | tr -s ' '` @@ -13,12 +13,14 @@ cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read _line; do continue fi - export ARCH=`echo $line | cut -d ' ' -f 1` - export ARCH_LABEL=`echo $line | cut -d ' ' -f 2` - export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 3` - export TIMEOUT=`echo $line | cut -d ' ' -f 4` - export ROOTFS_TAG=`echo $line | cut -d ' ' -f 5` - export ROOTFS_TREE=`echo $line | cut -d ' ' -f 6` - echo "Launching: $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $TIMEOUT" - buildkite-agent pipeline upload "$SCRIPT_DIR/$PLATFORM.yml" + export PLATFORM=`echo $line | cut -d ' ' -f 1` + export ARCH=`echo $line | cut -d ' ' -f 2` + export ARCH_LABEL=`echo $line | cut -d ' ' -f 3` + export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 4` + export ALLOW_FAIL=`echo $line | cut -d ' ' -f 5` + export TIMEOUT=`echo $line | cut -d ' ' -f 6` + export ROOTFS_TAG=`echo $line | cut -d ' ' -f 7` + export ROOTFS_TREE=`echo $line | cut -d ' ' -f 8` + echo "Launching: $OS $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $ALLOW_FAIL $TIMEOUT" + buildkite-agent pipeline upload "$SCRIPT_DIR/$OS.yml" done diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches index 7d8e8ba607533..33216f39a5eb8 100644 --- a/.buildkite/pipelines/main/platforms/tester_linux.arches +++ b/.buildkite/pipelines/main/platforms/tester_linux.arches @@ -1,8 +1,9 @@ -# ARCH ARCH_LABEL ROOTFS_ARCH TIMEOUT ROOTFS_TAG ROOTFS_TREE -# aarch64 aarch64 aarch64 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff -# armv7l armv7l armv7l 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 -32 32 i686 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 -# ppc64le ppc64le powerpc64le 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 -64 64_rr x86_64 180 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 -64 64_st x86_64 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 -64 64_mt x86_64 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +# PLATFORM ARCH ARCH_LABEL ROOTFS_ARCH ALLOW_FAIL TIMEOUT ROOTFS_TAG ROOTFS_TREE +# linux aarch64 aarch64 aarch64 false 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff +# linux armv7l armv7l armv7l false 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 +linux 32 32 i686 false 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 +# linux ppc64le ppc64le powerpc64le false 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 +linux 64 64_rr x86_64 false 180 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +linux 64 64_st x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +linux 64 64_mt x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 +musl 64 64 x86_64 true 60 v3.19 e6a2730e37c386c46915b2650d6aaaa398195152 diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml index 295de3b881054..c6e6c072904c1 100644 --- a/.buildkite/pipelines/main/platforms/tester_linux.yml +++ b/.buildkite/pipelines/main/platforms/tester_linux.yml @@ -4,16 +4,17 @@ agents: sandbox.jl: "true" os: "linux" steps: - - label: "tester_linux${ARCH_LABEL?}" - key: tester_linux${ARCH_LABEL?} - depends_on: package_linux${ARCH?} + - label: "tester_${PLATFORM?}${ARCH_LABEL?}" + key: tester_${PLATFORM?}${ARCH_LABEL?} + depends_on: package_${PLATFORM?}${ARCH?} plugins: - JuliaCI/julia#v1: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz rootfs_treehash: "${ROOTFS_TREE?}" uid: 1000 gid: 1000 @@ -23,6 +24,7 @@ steps: env: JULIA_SHELL: "/bin/bash" timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} commands: | echo "--- Print the full and short commit hashes" SHORT_COMMIT_LENGTH=10 @@ -30,7 +32,7 @@ steps: JULIA_DIR="julia-$${SHORT_COMMIT:?}" JULIA_BINARY="$${JULIA_DIR:?}/bin/julia" ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-linux${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" + ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" echo "The full commit is: $${BUILDKITE_COMMIT:?}" echo "The short commit is: $${SHORT_COMMIT:?}" echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}"