Skip to content

Commit 2dcf54f

Browse files
borsbors
committed
Auto merge of #70190 - pietroalbini:gha, r=Mark-Simulacrum
Add GitHub Actions configuration This PR adds the GitHub Actions configuration to the rust-lang/rust repository. The configuration will be run in parallel with Azure Pipelines until the evaluation finishes: the infrastructure team will then decide whether to switch. Since GitHub Actions doesn't currently have any way to include pieces of configuration, this also adds the `src/tools/expand-yaml-anchors` tool, which serves as a sort of templating system. Otherwise the configuration is a mostly straight port from the Azure Pipelines configuration (thanks to all the PRs opened in the past). There are still a few small things I need to fix before we can land this, but it's mostly complete and ready for an initial review. r? @Mark-Simulacrum
2 parents 374ab25 + 45910e7 commit 2dcf54f

25 files changed

+1994
-30
lines changed

‎.github/workflows/ci.yml

+781
Large diffs are not rendered by default.

‎Cargo.lock

+55-1
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,7 @@ dependencies = [
443443
"textwrap",
444444
"unicode-width",
445445
"vec_map",
446-
"yaml-rust",
446+
"yaml-rust 0.3.5",
447447
]
448448

449449
[[package]]
@@ -1026,6 +1026,14 @@ dependencies = [
10261026
"walkdir",
10271027
]
10281028

1029+
[[package]]
1030+
name = "expand-yaml-anchors"
1031+
version = "0.1.0"
1032+
dependencies = [
1033+
"yaml-merge-keys",
1034+
"yaml-rust 0.4.3",
1035+
]
1036+
10291037
[[package]]
10301038
name = "failure"
10311039
version = "0.1.5"
@@ -1830,6 +1838,12 @@ dependencies = [
18301838
name = "linkchecker"
18311839
version = "0.1.0"
18321840

1841+
[[package]]
1842+
name = "linked-hash-map"
1843+
version = "0.5.2"
1844+
source = "registry+https://github.com/rust-lang/crates.io-index"
1845+
checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
1846+
18331847
[[package]]
18341848
name = "lock_api"
18351849
version = "0.3.1"
@@ -4854,6 +4868,26 @@ dependencies = [
48544868
"unicode-width",
48554869
]
48564870

4871+
[[package]]
4872+
name = "thiserror"
4873+
version = "1.0.5"
4874+
source = "registry+https://github.com/rust-lang/crates.io-index"
4875+
checksum = "f9fb62ff737e573b1e677459bea6fd023cd5d6e868c3242d3cdf3ef2f0554824"
4876+
dependencies = [
4877+
"thiserror-impl",
4878+
]
4879+
4880+
[[package]]
4881+
name = "thiserror-impl"
4882+
version = "1.0.5"
4883+
source = "registry+https://github.com/rust-lang/crates.io-index"
4884+
checksum = "24069c0ba08aab54289d6a25f5036d94afc61e1538bbc42ae5501df141c9027d"
4885+
dependencies = [
4886+
"proc-macro2 1.0.3",
4887+
"quote 1.0.2",
4888+
"syn 1.0.11",
4889+
]
4890+
48574891
[[package]]
48584892
name = "thread_local"
48594893
version = "0.3.6"
@@ -5536,8 +5570,28 @@ dependencies = [
55365570
"lzma-sys",
55375571
]
55385572

5573+
[[package]]
5574+
name = "yaml-merge-keys"
5575+
version = "0.4.0"
5576+
source = "registry+https://github.com/rust-lang/crates.io-index"
5577+
checksum = "59893318ba3ad2b704498c7761214a10eaf42c5f838bce9fc0145bf2ba658cfa"
5578+
dependencies = [
5579+
"lazy_static 1.4.0",
5580+
"thiserror",
5581+
"yaml-rust 0.4.3",
5582+
]
5583+
55395584
[[package]]
55405585
name = "yaml-rust"
55415586
version = "0.3.5"
55425587
source = "registry+https://github.com/rust-lang/crates.io-index"
55435588
checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
5589+
5590+
[[package]]
5591+
name = "yaml-rust"
5592+
version = "0.4.3"
5593+
source = "registry+https://github.com/rust-lang/crates.io-index"
5594+
checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d"
5595+
dependencies = [
5596+
"linked-hash-map",
5597+
]

‎Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ members = [
2424
"src/tools/miri",
2525
"src/tools/rustdoc-themes",
2626
"src/tools/unicode-table-generator",
27+
"src/tools/expand-yaml-anchors",
2728
]
2829
exclude = [
2930
"build",

‎src/bootstrap/builder.rs

+5
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use crate::doc;
2121
use crate::flags::Subcommand;
2222
use crate::install;
2323
use crate::native;
24+
use crate::run;
2425
use crate::test;
2526
use crate::tool;
2627
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir};
@@ -313,6 +314,7 @@ pub enum Kind {
313314
Dist,
314315
Doc,
315316
Install,
317+
Run,
316318
}
317319

318320
impl<'a> Builder<'a> {
@@ -353,6 +355,7 @@ impl<'a> Builder<'a> {
353355
}
354356
Kind::Test => describe!(
355357
crate::toolstate::ToolStateCheck,
358+
test::ExpandYamlAnchors,
356359
test::Tidy,
357360
test::Ui,
358361
test::CompileFail,
@@ -454,6 +457,7 @@ impl<'a> Builder<'a> {
454457
install::Src,
455458
install::Rustc
456459
),
460+
Kind::Run => describe!(run::ExpandYamlAnchors,),
457461
}
458462
}
459463

@@ -507,6 +511,7 @@ impl<'a> Builder<'a> {
507511
Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
508512
Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
509513
Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
514+
Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
510515
Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(),
511516
};
512517

‎src/bootstrap/flags.rs

+24
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ pub enum Subcommand {
8686
Install {
8787
paths: Vec<PathBuf>,
8888
},
89+
Run {
90+
paths: Vec<PathBuf>,
91+
},
8992
}
9093

9194
impl Default for Subcommand {
@@ -113,6 +116,7 @@ Subcommands:
113116
clean Clean out build directories
114117
dist Build distribution artifacts
115118
install Install distribution artifacts
119+
run Run tools contained in this repository
116120
117121
To learn more about a subcommand, run `./x.py <subcommand> -h`",
118122
);
@@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
188192
|| (s == "clean")
189193
|| (s == "dist")
190194
|| (s == "install")
195+
|| (s == "run")
191196
});
192197
let subcommand = match subcommand {
193198
Some(s) => s,
@@ -400,6 +405,18 @@ Arguments:
400405
./x.py doc --stage 1",
401406
);
402407
}
408+
"run" => {
409+
subcommand_help.push_str(
410+
"\n
411+
Arguments:
412+
This subcommand accepts a number of paths to tools to build and run. For
413+
example:
414+
415+
./x.py run src/tool/expand-yaml-anchors
416+
417+
At least a tool needs to be called.",
418+
);
419+
}
403420
_ => {}
404421
};
405422
// Get any optional paths which occur after the subcommand
@@ -468,6 +485,13 @@ Arguments:
468485
"fmt" => Subcommand::Format { check: matches.opt_present("check") },
469486
"dist" => Subcommand::Dist { paths },
470487
"install" => Subcommand::Install { paths },
488+
"run" => {
489+
if paths.is_empty() {
490+
println!("\nrun requires at least a path!\n");
491+
usage(1, &opts, &subcommand_help, &extra_help);
492+
}
493+
Subcommand::Run { paths }
494+
}
471495
_ => {
472496
usage(1, &opts, &subcommand_help, &extra_help);
473497
}

‎src/bootstrap/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ mod format;
140140
mod install;
141141
mod metadata;
142142
mod native;
143+
mod run;
143144
mod sanity;
144145
mod test;
145146
mod tool;

‎src/bootstrap/run.rs

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
2+
use crate::tool::Tool;
3+
use std::process::Command;
4+
5+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
6+
pub struct ExpandYamlAnchors;
7+
8+
impl Step for ExpandYamlAnchors {
9+
type Output = ();
10+
11+
/// Runs the `expand-yaml_anchors` tool.
12+
///
13+
/// This tool in `src/tools` read the CI configuration files written in YAML and expands the
14+
/// anchors in them, since GitHub Actions doesn't support them.
15+
fn run(self, builder: &Builder<'_>) {
16+
builder.info("Expanding YAML anchors in the GitHub Actions configuration");
17+
try_run(
18+
builder,
19+
&mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
20+
);
21+
}
22+
23+
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
24+
run.path("src/tools/expand-yaml-anchors")
25+
}
26+
27+
fn make_run(run: RunConfig<'_>) {
28+
run.builder.ensure(ExpandYamlAnchors);
29+
}
30+
}
31+
32+
fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
33+
if !builder.fail_fast {
34+
if !builder.try_run(cmd) {
35+
let mut failures = builder.delayed_failures.borrow_mut();
36+
failures.push(format!("{:?}", cmd));
37+
return false;
38+
}
39+
} else {
40+
builder.run(cmd);
41+
}
42+
true
43+
}

‎src/bootstrap/test.rs

+29
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,35 @@ impl Step for Tidy {
750750
}
751751
}
752752

753+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
754+
pub struct ExpandYamlAnchors;
755+
756+
impl Step for ExpandYamlAnchors {
757+
type Output = ();
758+
const ONLY_HOSTS: bool = true;
759+
760+
/// Ensure the `generate-ci-config` tool was run locally.
761+
///
762+
/// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
763+
/// appropriate configuration for all our CI providers. This step ensures the tool was called
764+
/// by the user before committing CI changes.
765+
fn run(self, builder: &Builder<'_>) {
766+
builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
767+
try_run(
768+
builder,
769+
&mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
770+
);
771+
}
772+
773+
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
774+
run.path("src/tools/expand-yaml-anchors")
775+
}
776+
777+
fn make_run(run: RunConfig<'_>) {
778+
run.builder.ensure(ExpandYamlAnchors);
779+
}
780+
}
781+
753782
fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf {
754783
builder.out.join(host).join("test")
755784
}

‎src/bootstrap/tool.rs

+1
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,7 @@ bootstrap_tool!(
378378
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
379379
RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
380380
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
381+
ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
381382
);
382383

383384
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

‎src/ci/azure-pipelines/auto.yml

+11
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
#####################################
2+
## READ BEFORE CHANGING THIS ##
3+
#####################################
4+
5+
# We're in the process of evaluating GitHub Actions as a possible replacement
6+
# for Azure Pipelines, and at the moment the configuration is duplicated
7+
# between the two CI providers. Be sure to also change the configuration in
8+
# src/ci/github-actions when changing this file.
9+
10+
#####################################
11+
112
#
213
# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
314
#

‎src/ci/azure-pipelines/master.yml

+11
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
#####################################
2+
## READ BEFORE CHANGING THIS ##
3+
#####################################
4+
5+
# We're in the process of evaluating GitHub Actions as a possible replacement
6+
# for Azure Pipelines, and at the moment the configuration is duplicated
7+
# between the two CI providers. Be sure to also change the configuration in
8+
# src/ci/github-actions when changing this file.
9+
10+
#####################################
11+
112
#
213
# Azure Pipelines job to publish toolstate. Only triggers on pushes to master.
314
#

‎src/ci/azure-pipelines/pr.yml

+11
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
#####################################
2+
## READ BEFORE CHANGING THIS ##
3+
#####################################
4+
5+
# We're in the process of evaluating GitHub Actions as a possible replacement
6+
# for Azure Pipelines, and at the moment the configuration is duplicated
7+
# between the two CI providers. Be sure to also change the configuration in
8+
# src/ci/github-actions when changing this file.
9+
10+
#####################################
11+
112
#
213
# Azure Pipelines pull request build for Rust
314
#

‎src/ci/azure-pipelines/steps/run.yml

+13-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
#####################################
2+
## READ BEFORE CHANGING THIS ##
3+
#####################################
4+
5+
# We're in the process of evaluating GitHub Actions as a possible replacement
6+
# for Azure Pipelines, and at the moment the configuration is duplicated
7+
# between the two CI providers. Be sure to also change the configuration in
8+
# src/ci/github-actions when changing this file.
9+
10+
#####################################
11+
112
# FIXME(linux): need to configure core dumps, enable them, and then dump
213
# backtraces on failure from all core dumps:
314
#
@@ -59,8 +70,8 @@ steps:
5970
displayName: Install InnoSetup
6071
condition: and(succeeded(), not(variables.SKIP_JOB))
6172

62-
- bash: src/ci/scripts/windows-symlink-build-dir.sh
63-
displayName: Ensure the build happens on C:\ instead of D:\
73+
- bash: src/ci/scripts/symlink-build-dir.sh
74+
displayName: Ensure the build happens on a partition with enough space
6475
condition: and(succeeded(), not(variables.SKIP_JOB))
6576

6677
- bash: src/ci/scripts/disable-git-crlf-conversion.sh

‎src/ci/azure-pipelines/try.yml

+11
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
#####################################
2+
## READ BEFORE CHANGING THIS ##
3+
#####################################
4+
5+
# We're in the process of evaluating GitHub Actions as a possible replacement
6+
# for Azure Pipelines, and at the moment the configuration is duplicated
7+
# between the two CI providers. Be sure to also change the configuration in
8+
# src/ci/github-actions when changing this file.
9+
10+
#####################################
11+
112
pr: none
213
trigger:
314
- try

‎src/ci/docker/mingw-check/Dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ RUN sh /scripts/sccache.sh
2222
COPY mingw-check/validate-toolstate.sh /scripts/
2323

2424
ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
25-
ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
25+
ENV SCRIPT python2.7 ../x.py test src/tools/expand-yaml-anchors && \
26+
python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
2627
python2.7 ../x.py build --stage 0 src/tools/build-manifest && \
2728
python2.7 ../x.py test --stage 0 src/tools/compiletest && \
2829
python2.7 ../x.py test src/tools/tidy && \

0 commit comments

Comments
 (0)