Auto merge of #101799 - LukeMathWalker:distribute-json-doc, r=jyn514

Distribute json doc

# Overview

We add a new component, `rust-json-docs`, to distribute the JSON version of rustdoc's output for public compiler crates (i.e. `std`, `alloc`, `proc_macro`, `core` and `test`).
As discussed in #101383, we do not bundle this up as part of the existing `rust-docs` component since `rustdoc`'s JSON format is still unstable.

# Open questions / Doubts

I tried my best, but I never touched this codebase and I couldn't find much documentation on how `dist` works - I pattern-matched existing code, which might have led to some non-sensical choices in the eyes of people more familiar with the codebase. In particular, I am not sure if my choice of adding a new config flag is appropriate or if the decision to build/not build the JSON docs is more appropriately gated by one of the existing flags.
Any suggestion is more than welcome.

Closes #101383

src/bootstrap/builder.rs

@@ -708,6 +708,7 @@ impl<'a> Builder<'a> {
             Kind::Dist => describe!(
                 dist::Docs,
                 dist::RustcDocs,
+                dist::JsonDocs,
                 dist::Mingw,
                 dist::Rustc,
                 dist::Std,

src/bootstrap/builder/tests.rs

@@ -236,7 +236,7 @@ mod defaults {
     fn doc_default() {
         let mut config = configure("doc", &["A"], &["A"]);
         config.compiler_docs = true;
-        config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
+        config.cmd = Subcommand::Doc { paths: Vec::new(), open: false, json: false };
         let mut cache = run_build(&[], config);
         let a = TargetSelection::from_user("A");
 
@@ -587,7 +587,7 @@ mod dist {
     fn doc_ci() {
         let mut config = configure(&["A"], &["A"]);
         config.compiler_docs = true;
-        config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
+        config.cmd = Subcommand::Doc { paths: Vec::new(), open: false, json: false };
         let build = Build::new(config);
         let mut builder = Builder::new(&build);
         builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]);

src/bootstrap/dist.rs

@@ -87,6 +87,45 @@ impl Step for Docs {
     }
 }
 
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct JsonDocs {
+    pub host: TargetSelection,
+}
+
+impl Step for JsonDocs {
+    type Output = Option<GeneratedTarball>;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let default = run.builder.config.docs;
+        run.alias("rust-json-docs").default_condition(default)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(JsonDocs { host: run.target });
+    }
+
+    /// Builds the `rust-json-docs` installer component.
+    fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        // This prevents JSON docs from being built for "dist" or "install"
+        // on the stable/beta channels. The JSON format is not stable yet and
+        // should not be included in stable/beta toolchains.
+        if !builder.build.unstable_features() {
+            return None;
+        }
+
+        let host = self.host;
+        builder.ensure(crate::doc::JsonStd { stage: builder.top_stage, target: host });
+
+        let dest = "share/doc/rust/json";
+
+        let mut tarball = Tarball::new(builder, "rust-json-docs", &host.triple);
+        tarball.set_product_name("Rust Documentation In JSON Format");
+        tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
+        Some(tarball.generate())
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct RustcDocs {
     pub host: TargetSelection,

src/bootstrap/doc.rs

@@ -7,6 +7,7 @@
 //! Everything here is basically just a shim around calling either `rustbook` or
 //! `rustdoc`.
 
+use std::ffi::OsStr;
 use std::fs;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -425,49 +426,24 @@ impl Step for Std {
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let target = self.target;
-        builder.info(&format!("Documenting stage{} std ({})", stage, target));
-        if builder.no_std(target) == Some(true) {
-            panic!(
-                "building std documentation for no_std target {target} is not supported\n\
-                 Set `docs = false` in the config to disable documentation."
-            );
-        }
         let out = builder.doc_out(target);
         t!(fs::create_dir_all(&out));
-        let compiler = builder.compiler(stage, builder.config.build);
-
-        let out_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join("doc");
-
         t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css")));
 
-        let run_cargo_rustdoc_for = |package: &str| {
-            let mut cargo =
-                builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
-            compile::std_cargo(builder, target, compiler.stage, &mut cargo);
-
-            cargo
-                .arg("-p")
-                .arg(package)
-                .arg("-Zskip-rustdoc-fingerprint")
-                .arg("--")
-                .arg("--markdown-css")
-                .arg("rust.css")
-                .arg("--markdown-no-toc")
-                .arg("-Z")
-                .arg("unstable-options")
-                .arg("--resource-suffix")
-                .arg(&builder.version)
-                .arg("--index-page")
-                .arg(&builder.src.join("src/doc/index.md"));
-
-            if !builder.config.docs_minification {
-                cargo.arg("--disable-minification");
-            }
-
-            builder.run(&mut cargo.into());
-        };
+        let index_page = builder.src.join("src/doc/index.md").into_os_string();
+        let mut extra_args = vec![
+            OsStr::new("--markdown-css"),
+            OsStr::new("rust.css"),
+            OsStr::new("--markdown-no-toc"),
+            OsStr::new("--index-page"),
+            &index_page,
+        ];
+
+        if !builder.config.docs_minification {
+            extra_args.push(OsStr::new("--disable-minification"));
+        }
 
-        let paths = builder
+        let requested_crates = builder
             .paths
             .iter()
             .map(components_simplified)
@@ -485,37 +461,155 @@ impl Step for Std {
             })
             .collect::<Vec<_>>();
 
-        // Only build the following crates. While we could just iterate over the
-        // folder structure, that would also build internal crates that we do
-        // not want to show in documentation. These crates will later be visited
-        // by the rustc step, so internal documentation will show them.
-        //
-        // Note that the order here is important! The crates need to be
-        // processed starting from the leaves, otherwise rustdoc will not
-        // create correct links between crates because rustdoc depends on the
-        // existence of the output directories to know if it should be a local
-        // or remote link.
-        let krates = ["core", "alloc", "std", "proc_macro", "test"];
-        for krate in &krates {
-            run_cargo_rustdoc_for(krate);
-            if paths.iter().any(|p| p == krate) {
-                // No need to document more of the libraries if we have the one we want.
-                break;
-            }
-        }
-        builder.cp_r(&out_dir, &out);
+        doc_std(
+            builder,
+            DocumentationFormat::HTML,
+            stage,
+            target,
+            &out,
+            &extra_args,
+            &requested_crates,
+        );
 
         // Look for library/std, library/core etc in the `x.py doc` arguments and
         // open the corresponding rendered docs.
-        for requested_crate in paths {
-            if krates.iter().any(|k| *k == requested_crate.as_str()) {
+        for requested_crate in requested_crates {
+            if STD_PUBLIC_CRATES.iter().any(|k| *k == requested_crate.as_str()) {
                 let index = out.join(requested_crate).join("index.html");
                 open(builder, &index);
             }
         }
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct JsonStd {
+    pub stage: u32,
+    pub target: TargetSelection,
+}
+
+impl Step for JsonStd {
+    type Output = ();
+    const DEFAULT: bool = false;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let default = run.builder.config.docs && run.builder.config.cmd.json();
+        run.all_krates("test").path("library").default_condition(default)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Std { stage: run.builder.top_stage, target: run.target });
+    }
+
+    /// Build JSON documentation for the standard library crates.
+    ///
+    /// This is largely just a wrapper around `cargo doc`.
+    fn run(self, builder: &Builder<'_>) {
+        let stage = self.stage;
+        let target = self.target;
+        let out = builder.json_doc_out(target);
+        t!(fs::create_dir_all(&out));
+        let extra_args = [OsStr::new("--output-format"), OsStr::new("json")];
+        doc_std(builder, DocumentationFormat::JSON, stage, target, &out, &extra_args, &[])
+    }
+}
+
+/// Name of the crates that are visible to consumers of the standard library.
+/// Documentation for internal crates is handled by the rustc step, so internal crates will show
+/// up there.
+///
+/// Order here is important!
+/// Crates need to be processed starting from the leaves, otherwise rustdoc will not
+/// create correct links between crates because rustdoc depends on the
+/// existence of the output directories to know if it should be a local
+/// or remote link.
+const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"];
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+enum DocumentationFormat {
+    HTML,
+    JSON,
+}
+
+impl DocumentationFormat {
+    fn as_str(&self) -> &str {
+        match self {
+            DocumentationFormat::HTML => "HTML",
+            DocumentationFormat::JSON => "JSON",
+        }
+    }
+}
+
+/// Build the documentation for public standard library crates.
+///
+/// `requested_crates` can be used to build only a subset of the crates. If empty, all crates will
+/// be built.
+fn doc_std(
+    builder: &Builder<'_>,
+    format: DocumentationFormat,
+    stage: u32,
+    target: TargetSelection,
+    out: &Path,
+    extra_args: &[&OsStr],
+    requested_crates: &[String],
+) {
+    builder.info(&format!(
+        "Documenting stage{} std ({}) in {} format",
+        stage,
+        target,
+        format.as_str()
+    ));
+    if builder.no_std(target) == Some(true) {
+        panic!(
+            "building std documentation for no_std target {target} is not supported\n\
+             Set `docs = false` in the config to disable documentation."
+        );
+    }
+    let compiler = builder.compiler(stage, builder.config.build);
+    // This is directory where the compiler will place the output of the command.
+    // We will then copy the files from this directory into the final `out` directory, the specified
+    // as a function parameter.
+    let out_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join("doc");
+    // `cargo` uses the same directory for both JSON docs and HTML docs.
+    // This could lead to cross-contamination when copying files into the specified `out` directory.
+    // For example:
+    // ```bash
+    // x doc std
+    // x doc std --json
+    // ```
+    // could lead to HTML docs being copied into the JSON docs output directory.
+    // To avoid this issue, we clean the doc folder before invoking `cargo`.
+    if out_dir.exists() {
+        builder.remove_dir(&out_dir);
+    }
+
+    let run_cargo_rustdoc_for = |package: &str| {
+        let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
+        compile::std_cargo(builder, target, compiler.stage, &mut cargo);
+        cargo
+            .arg("-p")
+            .arg(package)
+            .arg("-Zskip-rustdoc-fingerprint")
+            .arg("--")
+            .arg("-Z")
+            .arg("unstable-options")
+            .arg("--resource-suffix")
+            .arg(&builder.version)
+            .args(extra_args);
+        builder.run(&mut cargo.into());
+    };
+
+    for krate in STD_PUBLIC_CRATES {
+        run_cargo_rustdoc_for(krate);
+        if requested_crates.iter().any(|p| p == krate) {
+            // No need to document more of the libraries if we have the one we want.
+            break;
+        }
+    }
+
+    builder.cp_r(&out_dir, &out);
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustc {
     pub stage: u32,

src/bootstrap/flags.rs

@@ -107,6 +107,7 @@ pub enum Subcommand {
     Doc {
         paths: Vec<PathBuf>,
         open: bool,
+        json: bool,
     },
     Test {
         paths: Vec<PathBuf>,
@@ -325,6 +326,11 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
             }
             Kind::Doc => {
                 opts.optflag("", "open", "open the docs in a browser");
+                opts.optflag(
+                    "",
+                    "json",
+                    "render the documentation in JSON format in addition to the usual HTML format",
+                );
             }
             Kind::Clean => {
                 opts.optflag("", "all", "clean all build artifacts");
@@ -493,6 +499,7 @@ Arguments:
         ./x.py doc src/doc/book
         ./x.py doc src/doc/nomicon
         ./x.py doc src/doc/book library/std
+        ./x.py doc library/std --json
         ./x.py doc library/std --open
 
     If no arguments are passed then everything is documented:
@@ -581,7 +588,11 @@ Arguments:
                 },
             },
             Kind::Bench => Subcommand::Bench { paths, test_args: matches.opt_strs("test-args") },
-            Kind::Doc => Subcommand::Doc { paths, open: matches.opt_present("open") },
+            Kind::Doc => Subcommand::Doc {
+                paths,
+                open: matches.opt_present("open"),
+                json: matches.opt_present("json"),
+            },
             Kind::Clean => {
                 if !paths.is_empty() {
                     println!("\nclean does not take a path argument\n");
@@ -787,6 +798,13 @@ impl Subcommand {
             _ => false,
         }
     }
+
+    pub fn json(&self) -> bool {
+        match *self {
+            Subcommand::Doc { json, .. } => json,
+            _ => false,
+        }
+    }
 }
 
 fn split(s: &[String]) -> Vec<String> {

src/bootstrap/lib.rs

@@ -825,6 +825,11 @@ impl Build {
         self.out.join(&*target.triple).join("doc")
     }
 
+    /// Output directory for all JSON-formatted documentation for a target
+    fn json_doc_out(&self, target: TargetSelection) -> PathBuf {
+        self.out.join(&*target.triple).join("json-doc")
+    }
+
     fn test_out(&self, target: TargetSelection) -> PathBuf {
         self.out.join(&*target.triple).join("test")
     }