Auto merge of #70190 - pietroalbini:gha, r=Mark-Simulacrum

Add GitHub Actions configuration This PR adds the GitHub Actions configuration to the rust-lang/rust repository. The configuration will be run in parallel with Azure Pipelines until the evaluation finishes: the infrastructure team will then decide whether to switch. Since GitHub Actions doesn't currently have any way to include pieces of configuration, this also adds the `src/tools/expand-yaml-anchors` tool, which serves as a sort of templating system. Otherwise the configuration is a mostly straight port from the Azure Pipelines configuration (thanks to all the PRs opened in the past). There are still a few small things I need to fix before we can land this, but it's mostly complete and ready for an initial review. r? @Mark-Simulacrum
rust-lang · Mar 24, 2020 · 2dcf54f · 2dcf54f
2 parents 374ab25 + 45910e7
commit 2dcf54f
Show file tree

Hide file tree

Showing 25 changed files with 1,994 additions and 30 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
diff --git a/Cargo.lock b/Cargo.lock
@@ -443,7 +443,7 @@ dependencies = [
  "textwrap",
  "unicode-width",
  "vec_map",
- "yaml-rust",
+ "yaml-rust 0.3.5",
 ]
 
 [[package]]
@@ -1026,6 +1026,14 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "expand-yaml-anchors"
+version = "0.1.0"
+dependencies = [
+ "yaml-merge-keys",
+ "yaml-rust 0.4.3",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.5"
@@ -1830,6 +1838,12 @@ dependencies = [
 name = "linkchecker"
 version = "0.1.0"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
+
 [[package]]
 name = "lock_api"
 version = "0.3.1"
@@ -4854,6 +4868,26 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9fb62ff737e573b1e677459bea6fd023cd5d6e868c3242d3cdf3ef2f0554824"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24069c0ba08aab54289d6a25f5036d94afc61e1538bbc42ae5501df141c9027d"
+dependencies = [
+ "proc-macro2 1.0.3",
+ "quote 1.0.2",
+ "syn 1.0.11",
+]
+
 [[package]]
 name = "thread_local"
 version = "0.3.6"
@@ -5536,8 +5570,28 @@ dependencies = [
  "lzma-sys",
 ]
 
+[[package]]
+name = "yaml-merge-keys"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59893318ba3ad2b704498c7761214a10eaf42c5f838bce9fc0145bf2ba658cfa"
+dependencies = [
+ "lazy_static 1.4.0",
+ "thiserror",
+ "yaml-rust 0.4.3",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d"
+dependencies = [
+ "linked-hash-map",
+]
diff --git a/Cargo.toml b/Cargo.toml
@@ -24,6 +24,7 @@ members = [
   "src/tools/miri",
   "src/tools/rustdoc-themes",
   "src/tools/unicode-table-generator",
+  "src/tools/expand-yaml-anchors",
 ]
 exclude = [
   "build",

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
@@ -21,6 +21,7 @@ use crate::doc;
 use crate::flags::Subcommand;
 use crate::install;
 use crate::native;
+use crate::run;
 use crate::test;
 use crate::tool;
 use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir};
@@ -313,6 +314,7 @@ pub enum Kind {
     Dist,
     Doc,
     Install,
+    Run,
 }
 
 impl<'a> Builder<'a> {
@@ -353,6 +355,7 @@ impl<'a> Builder<'a> {
             }
             Kind::Test => describe!(
                 crate::toolstate::ToolStateCheck,
+                test::ExpandYamlAnchors,
                 test::Tidy,
                 test::Ui,
                 test::CompileFail,
@@ -454,6 +457,7 @@ impl<'a> Builder<'a> {
                 install::Src,
                 install::Rustc
             ),
+            Kind::Run => describe!(run::ExpandYamlAnchors,),
         }
     }
 
@@ -507,6 +511,7 @@ impl<'a> Builder<'a> {
             Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
+            Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
             Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(),
         };
 

diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
@@ -86,6 +86,9 @@ pub enum Subcommand {
     Install {
         paths: Vec<PathBuf>,
     },
+    Run {
+        paths: Vec<PathBuf>,
+    },
 }
 
 impl Default for Subcommand {
@@ -113,6 +116,7 @@ Subcommands:
     clean       Clean out build directories
     dist        Build distribution artifacts
     install     Install distribution artifacts
+    run         Run tools contained in this repository
 
 To learn more about a subcommand, run `./x.py <subcommand> -h`",
         );
@@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
                 || (s == "clean")
                 || (s == "dist")
                 || (s == "install")
+                || (s == "run")
         });
         let subcommand = match subcommand {
             Some(s) => s,
@@ -400,6 +405,18 @@ Arguments:
         ./x.py doc --stage 1",
                 );
             }
+            "run" => {
+                subcommand_help.push_str(
+                    "\n
+Arguments:
+    This subcommand accepts a number of paths to tools to build and run. For
+    example:
+
+        ./x.py run src/tool/expand-yaml-anchors
+
+    At least a tool needs to be called.",
+                );
+            }
             _ => {}
         };
         // Get any optional paths which occur after the subcommand
@@ -468,6 +485,13 @@ Arguments:
             "fmt" => Subcommand::Format { check: matches.opt_present("check") },
             "dist" => Subcommand::Dist { paths },
             "install" => Subcommand::Install { paths },
+            "run" => {
+                if paths.is_empty() {
+                    println!("\nrun requires at least a path!\n");
+                    usage(1, &opts, &subcommand_help, &extra_help);
+                }
+                Subcommand::Run { paths }
+            }
             _ => {
                 usage(1, &opts, &subcommand_help, &extra_help);
             }

diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
@@ -140,6 +140,7 @@ mod format;
 mod install;
 mod metadata;
 mod native;
+mod run;
 mod sanity;
 mod test;
 mod tool;

diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs
@@ -0,0 +1,43 @@
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::tool::Tool;
+use std::process::Command;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+    type Output = ();
+
+    /// Runs the `expand-yaml_anchors` tool.
+    ///
+    /// This tool in `src/tools` read the CI configuration files written in YAML and expands the
+    /// anchors in them, since GitHub Actions doesn't support them.
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("Expanding YAML anchors in the GitHub Actions configuration");
+        try_run(
+            builder,
+            &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
+        );
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/expand-yaml-anchors")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(ExpandYamlAnchors);
+    }
+}
+
+fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
+    if !builder.fail_fast {
+        if !builder.try_run(cmd) {
+            let mut failures = builder.delayed_failures.borrow_mut();
+            failures.push(format!("{:?}", cmd));
+            return false;
+        }
+    } else {
+        builder.run(cmd);
+    }
+    true
+}
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
@@ -750,6 +750,35 @@ impl Step for Tidy {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    /// Ensure the `generate-ci-config` tool was run locally.
+    ///
+    /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
+    /// appropriate configuration for all our CI providers. This step ensures the tool was called
+    /// by the user before committing CI changes.
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
+        try_run(
+            builder,
+            &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
+        );
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/expand-yaml-anchors")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(ExpandYamlAnchors);
+    }
+}
+
 fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf {
     builder.out.join(host).join("test")
 }

diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
@@ -378,6 +378,7 @@ bootstrap_tool!(
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
     RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
+    ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
 #

diff --git a/src/ci/azure-pipelines/master.yml b/src/ci/azure-pipelines/master.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines job to publish toolstate. Only triggers on pushes to master.
 #

diff --git a/src/ci/azure-pipelines/pr.yml b/src/ci/azure-pipelines/pr.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines pull request build for Rust
 #

diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 # FIXME(linux): need to configure core dumps, enable them, and then dump
 # backtraces on failure from all core dumps:
 #
@@ -59,8 +70,8 @@ steps:
   displayName: Install InnoSetup
   condition: and(succeeded(), not(variables.SKIP_JOB))
 
-- bash: src/ci/scripts/windows-symlink-build-dir.sh
-  displayName: Ensure the build happens on C:\ instead of D:\
+- bash: src/ci/scripts/symlink-build-dir.sh
+  displayName: Ensure the build happens on a partition with enough space
   condition: and(succeeded(), not(variables.SKIP_JOB))
 
 - bash: src/ci/scripts/disable-git-crlf-conversion.sh

diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 pr: none
 trigger:
 - try

diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile
@@ -22,7 +22,8 @@ RUN sh /scripts/sccache.sh
 COPY mingw-check/validate-toolstate.sh /scripts/
 
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
-ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
+ENV SCRIPT python2.7 ../x.py test src/tools/expand-yaml-anchors && \
+           python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
            python2.7 ../x.py build --stage 0 src/tools/build-manifest && \
            python2.7 ../x.py test --stage 0 src/tools/compiletest && \
            python2.7 ../x.py test src/tools/tidy && \