Merge branch 'master' into master

.buildkite/README.md

@@ -0,0 +1,7 @@
+# Buildkite
+
+This directory contains the Buildkite configuration files for Base Julia CI.
+
+The rootfs image definitions are located in the [rootfs-images](https://github.com/JuliaCI/rootfs-images) repository.
+
+The documentation for the Base Julia CI setup is located in the [base-buildkite-docs](https://github.com/JuliaCI/base-buildkite-docs) repository.

.buildkite/cryptic_repo_keys/.gitignore

@@ -0,0 +1,5 @@
+# Ignore the unencrypted repo_key
+repo_key
+
+# Ignore any agent keys (public or private) we have stored
+agent_key*

.buildkite/cryptic_repo_keys/README.md

@@ -0,0 +1,6 @@
+## Cryptic repository keys
+
+This folder contains RSA-encrypted symmetric AES keys.
+These are used by buildkite agents to decrypt the secrets embedded within this repository.
+Each buildkite agent contains an RSA secret key that is used to unlock the symmetric AES key that was used to encrypt the secrets within this repository.
+For more information, see the [`cryptic` buildkite plugin repository](https://github.com/staticfloat/cryptic-buildkite-plugin).

.buildkite/pipelines/experimental/0_webui.yml

@@ -0,0 +1,22 @@
+# This file represents what is put into the webUI.
+# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
+# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
+agents:
+  queue: "julia"
+  sandbox.jl: "true"
+steps:
+  - label: ":unlock: Unlock secrets, launch pipelines"
+    plugins:
+      - staticfloat/cryptic#v1:
+          # Our list of pipelines that should be launched (but don't require a signature)
+          # These pipelines can be modified by any contributor and CI will still run.
+          # Build secrets will not be available in these pipelines (or their children)
+          # but some of our signed pipelines can wait upon the completion of these unsigned
+          # pipelines.
+          unsigned_pipelines:
+            - .buildkite/pipelines/experimental/launch_unsigned_builders.yml
+          # Our signed pipelines must have a `signature` or `signature_file` parameter that
+          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+          # signed_pipelines:
+          #   - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
+          #     signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature

.buildkite/pipelines/experimental/README.md

@@ -0,0 +1,7 @@
+## Experimental pipeline (`master` branch only)
+
+This is the [`julia-master->experimental`](https://buildkite.com/julialang/julia-master-experimental) pipeline.
+
+We use this pipeline for builders that are not yet stable enough to go into the main pipeline.
+
+These builders are triggered by GitHub webhook events, such as pushes and pull requests.

.buildkite/pipelines/experimental/launch_unsigned_builders.yml

@@ -0,0 +1,6 @@
+steps:
+  - label: ":buildkite: Launch unsigned pipelines"
+    commands: |
+      true
+    agents:
+      queue: julia

.buildkite/pipelines/main/0_webui.yml

@@ -0,0 +1,23 @@
+# This file represents what is put into the webUI.
+# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
+# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
+agents:
+  queue: "julia"
+  sandbox.jl: "true"
+steps:
+  - label: ":unlock: Unlock secrets, launch pipelines"
+    plugins:
+      - staticfloat/cryptici#v1:
+          # Our list of pipelines that should be launched (but don't require a signature)
+          # These pipelines can be modified by any contributor and CI will still run.
+          # Build secrets will not be available in these pipelines (or their children)
+          # but some of our signed pipelines can wait upon the completion of these unsigned
+          # pipelines.
+          unsigned_pipelines:
+            - .buildkite/pipelines/main/launch_unsigned_builders.yml
+
+          # Our signed pipelines must have a `signature` or `signature_file` parameter that
+          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+          signed_pipelines:
+            - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml
+              signature_file: .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature

.buildkite/pipelines/main/README.md

@@ -0,0 +1,15 @@
+## Main pipeline
+
+This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests.
+
+We have a different main pipeline for each permanent branch.
+
+For example:
+
+| Permanent Branch | Pipeline                                                                         |
+| ---------------- | -------------------------------------------------------------------------------- |
+| `master`         | [`julia-master`](https://buildkite.com/julialang/julia-master)                   |
+| `release-1.6`    | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) |
+| `release-1.7`    | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) |
+
+(This is not a complete list.)

.buildkite/pipelines/main/launch_unsigned_builders.yml

@@ -0,0 +1,33 @@
+# This file launches all the build jobs that _don't_ require secrets access.
+# These jobs can pass their output off to jobs that do require secrets access,
+# but those privileged steps require signing before they can be run.
+#
+# Yes, this is creating another layer of indirection; the flow now looks like:
+#
+#   [webui] -> launch_unsigned_builders.yml -> misc/whitespace.yml
+#
+# when we could theoretically just have the `webui` launch `misc/whitespace.yml`,
+# however this raises the bar for contributors to add new (unsigned) steps to
+# our CI configuration, so I'd rather live with an extra layer of indirection
+# and only need to touch the webui configuration when we need to alter
+# something about the privileged steps.
+
+steps:
+  - label: ":buildkite: Launch unsigned jobs"
+    commands: |
+      # Launch the miscellaneous jobs in alphabetical order.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
+
+      # Launch all of the platform jobs.
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
+
+      # Launch the `whitespace` job last. Uploading it last actually causes it to start
+      # first. We want this job to start first because we want it to finish as quickly
+      # as possible.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
+    agents:
+      queue: julia

.buildkite/pipelines/main/misc/doctest.yml

@@ -0,0 +1,34 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "doctest"
+    key: doctest
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1.2:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    commands: |
+      echo "--- Build Julia from source"
+      make --output-sync -j 6
+
+      echo "--- Print Julia version info"
+      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+
+      echo "--- Build Julia docs"
+      make docs
+
+      echo "--- Run Julia doctests"
+      JULIA_NUM_THREADS=1 make -C doc doctest=true
+    timeout_in_minutes: 45

.buildkite/pipelines/main/misc/embedding.yml

@@ -0,0 +1,31 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "embedding"
+    key: "embedding"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1.2:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    commands: |
+      prefix="/tmp/prefix"
+      echo "+++ Build julia, deploy to $${prefix:?}"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
+
+      embedding_output="/tmp/embedding-test"
+      echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
+      mkdir -p "$${embedding_output:?}"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
+    timeout_in_minutes: 60

.buildkite/pipelines/main/misc/llvmpasses.yml

@@ -0,0 +1,51 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "analyzegc"
+    key: "analyzegc"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1.2:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    commands: |
+      echo "--- Install in-tree LLVM dependencies"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
+      echo "+++ run clangsa/analyzegc"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc
+    timeout_in_minutes: 60
+  - label: "llvmpasses"
+    key: "llvmpasses"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1.2:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    commands: |
+      echo "--- make release"
+      # Enable Julia assertions: FORCE_ASSERTIONS=1
+      # Enable LLVM assertions:  LLVM_ASSERTIONS=1
+      export MAKE_ASSERT_FLAGS="FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0 $${MAKE_ASSERT_FLAGS:?}
+      echo "--- make src/install-analysis-deps"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps $${MAKE_ASSERT_FLAGS:?}
+      echo "+++ make test/llvmpasses"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses $${MAKE_ASSERT_FLAGS:?}
+    timeout_in_minutes: 60

.buildkite/pipelines/main/misc/sanitizers.yml

@@ -0,0 +1,47 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "asan"
+    key: "asan"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1.2:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 120
+    if: | # We only run the `asan` job on Julia 1.8 and later.
+      (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
+    soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed
+    commands: |
+      echo "--- Build julia-debug with ASAN"
+      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
+  - label: "tsan"
+    key: "tsan"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1.2:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 120
+    if: | # We only run the `tsan` job on Julia 1.8 and later.
+      (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
+    commands: |
+      echo "--- Build julia-debug runtime with TSAN"
+      contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug

.buildkite/pipelines/main/misc/signed_pipeline_test.yml

@@ -0,0 +1,18 @@
+agents:
+  queue: "julia"
+  os: "linux"
+
+## pipeline that showcases decryption of environment variable
+steps:
+  - label: ":lock: :rocket: Signed pipeline test"
+    # We must accept the signed job id secret in order to propagate secrets
+    env:
+      BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}
+    depends_on:
+    plugins:
+      - staticfloat/cryptic#v1:
+          variables:
+            - SECRET_KEY="U2FsdGVkX18tb7st0SuQAvh4Yv4xENxOAu8q9XkmOeDVKBNY4FngEwK3xmiKUqaS"
+    commands: |
+      echo "SECRET_KEY: $${SECRET_KEY}"
+

.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature

@@ -0,0 +1 @@
+Salted__���J0�Q?���rۀ�g�~�d��ۛŧ�ө��o���Ujʀ���p�)�$�U$����y@gZM}{�m��,۠�K��e�r�