Auto merge of #74922 - joshtriplett:ninja-by-default, r=Mark-Simulacrum

Set ninja=true by default

Ninja substantially improves LLVM build time. On a 96-way system, using
Make took 248s, and using Ninja took 161s, a 35% improvement.

We already require a variety of tools to build Rust. If someone wants to
build without Ninja (for instance, to minimize the set of packages
required to bootstrap a new target), they can easily set `ninja=false`
in `config.toml`.  Our defaults should help people build Rust (and LLVM)
faster, to speed up development.

.github/workflows/ci.yml

@@ -583,7 +583,7 @@ jobs:
           - name: dist-x86_64-apple
             env:
               SCRIPT: "./x.py dist"
-              RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc"
+              RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
               NO_LLVM_ASSERTIONS: 1
@@ -594,7 +594,7 @@ jobs:
           - name: dist-x86_64-apple-alt
             env:
               SCRIPT: "./x.py dist"
-              RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc"
+              RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
               NO_LLVM_ASSERTIONS: 1
@@ -604,7 +604,7 @@ jobs:
           - name: x86_64-apple
             env:
               SCRIPT: "./x.py --stage 2 test"
-              RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc"
+              RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.8
               MACOSX_STD_DEPLOYMENT_TARGET: 10.7

README.md

@@ -36,6 +36,7 @@ by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
    * `python` 3 or 2.7
    * GNU `make` 3.81 or later
    * `cmake` 3.4.3 or later
+   * `ninja`
    * `curl`
    * `git`
    * `ssl` which comes in `libssl-dev` or `openssl-devel`

config.toml.example

@@ -49,10 +49,8 @@
 # dynamic version to be available.
 #static-libstdcpp = false
 
-# Tell the LLVM build system to use Ninja instead of the platform default for
-# the generated build system. This can sometimes be faster than make, for
-# example.
-#ninja = false
+# Whether to use Ninja to build LLVM. This runs much faster than make.
+#ninja = true
 
 # LLVM targets to build support for.
 # Note: this is NOT related to Rust compilation targets. However, as Rust is

src/bootstrap/builder/tests.rs

@@ -8,6 +8,7 @@ fn configure(host: &[&str], target: &[&str]) -> Config {
     config.save_toolstates = None;
     config.skip_only_host_steps = false;
     config.dry_run = true;
+    config.ninja = false;
     // try to avoid spurious failures in dist where we create/delete each others file
     let dir = config
         .out

src/bootstrap/config.rs

@@ -450,6 +450,7 @@ impl Config {
     pub fn default_opts() -> Config {
         let mut config = Config::default();
         config.llvm_optimize = true;
+        config.ninja = true;
         config.llvm_version_check = true;
         config.backtrace = true;
         config.rust_optimize = true;

src/bootstrap/sanity.rs

@@ -100,8 +100,16 @@ pub fn check(build: &mut Build) {
         if build.config.ninja {
             // Some Linux distros rename `ninja` to `ninja-build`.
             // CMake can work with either binary name.
-            if cmd_finder.maybe_have("ninja-build").is_none() {
-                cmd_finder.must_have("ninja");
+            if cmd_finder.maybe_have("ninja-build").is_none()
+                && cmd_finder.maybe_have("ninja").is_none()
+            {
+                eprintln!(
+                    "
+Couldn't find required command: ninja
+You should install ninja, or set ninja=false in config.toml
+"
+                );
+                std::process::exit(1);
             }
         }
 

src/ci/azure-pipelines/auto.yml

@@ -37,7 +37,7 @@ jobs:
       # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
       x86_64-apple:
         SCRIPT: ./x.py --stage 2 test
-        INITIAL_RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        INITIAL_RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
         RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
         MACOSX_DEPLOYMENT_TARGET: 10.8
         MACOSX_STD_DEPLOYMENT_TARGET: 10.7
@@ -46,7 +46,7 @@ jobs:
 
       dist-x86_64-apple:
         SCRIPT: ./x.py dist
-        INITIAL_RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
+        INITIAL_RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
         RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
         MACOSX_DEPLOYMENT_TARGET: 10.7
         NO_LLVM_ASSERTIONS: 1
@@ -55,7 +55,7 @@ jobs:
 
       dist-x86_64-apple-alt:
         SCRIPT: ./x.py dist
-        INITIAL_RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
+        INITIAL_RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false
         RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
         MACOSX_DEPLOYMENT_TARGET: 10.7
         NO_LLVM_ASSERTIONS: 1

src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:20.04
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/armhf-gnu/Dockerfile

@@ -14,6 +14,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
       libc6-dev \
       libc6-dev-armhf-cross \
       make \
+      ninja-build \
       python3 \
       qemu-system-arm \
       xz-utils

src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/disabled/dist-powerpcspe-linux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/disabled/dist-sparc64-linux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/disabled/dist-x86_64-dragonfly/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/disabled/dist-x86_64-haiku/Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libcurl4-openssl-dev \
   libssl-dev \
   make \
+  ninja-build \
   nasm \
   pkg-config \
   python3 \

src/ci/docker/host-x86_64/disabled/riscv64gc-linux/Dockerfile

@@ -20,6 +20,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
     libc6-dev \
     libc6-dev-riscv64-cross \
     make \
+    ninja-build \
     patch \
     python3 \
     qemu-system-misc \

src/ci/docker/host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++-multilib \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-i686-freebsd/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:18.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   clang \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile

@@ -93,6 +93,7 @@ ENV RUST_CONFIGURE_ARGS \
       --enable-profiler \
       --set target.i686-unknown-linux-gnu.linker=clang \
       --build=i686-unknown-linux-gnu \
+      --set llvm.ninja=false \
       --set rust.jemalloc
 ENV SCRIPT python2.7 ../x.py dist --build $HOSTS --host $HOSTS --target $HOSTS
 ENV CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER=clang

src/ci/docker/host-x86_64/dist-mips-linux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-mips64-linux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-mips64el-linux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-mipsel-linux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-various-1/Dockerfile

@@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libncurses-dev \
   gawk \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-various-2/build-cloudabi-toolchain.sh

@@ -16,6 +16,7 @@ apt-get install -y --no-install-recommends \
   git \
   lld-5.0 \
   make \
+  ninja-build \
   python \
   sudo \
   xz-utils

src/ci/docker/host-x86_64/dist-x86_64-freebsd/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:18.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   clang \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -96,6 +96,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.x86_64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
       --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
       --set llvm.thin-lto=true \
+      --set llvm.ninja=false \
       --set rust.jemalloc
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang

src/ci/docker/host-x86_64/dist-x86_64-musl/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   wget \
   curl \

src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++-multilib \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/i686-gnu/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++-multilib \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:18.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/test-various/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:18.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/wasm32/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:19.10
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu-llvm-8/Dockerfile

@@ -4,6 +4,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   g++-arm-linux-gnueabi \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:16.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:19.10
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
+  ninja-build \
   file \
   curl \
   ca-certificates \

src/ci/docker/scripts/android-base-apt-get.sh

@@ -10,6 +10,7 @@ apt-get install -y --no-install-recommends \
   git \
   libssl-dev \
   make \
+  ninja-build \
   pkg-config \
   python3 \
   sudo \

src/ci/docker/scripts/cross-apt-packages.sh

@@ -17,6 +17,7 @@ apt-get update && apt-get install -y --no-install-recommends \
   libssl-dev \
   libtool-bin \
   make \
+  ninja-build \
   patch \
   pkg-config \
   python3 \