Auto merge of #69115 - ehuss:update-books, r=Dylan-DPC

Update books.

This required some changes in how the books are tested due to some changes in rust-lang/book.  It uses new syntax that is not compatible with bare `rustdoc --test`.  This changes it so that it uses rustbook to run the tests, which is essentially the same as `mdbook test`.

## reference

7 commits in 11e893f..64239df
2020-01-18 21:24:08 +0100 to 2020-02-10 19:05:13 +0100
- Update for nested receivers. (rust-lang/reference#724)
- clarify note re. leading `::` in 2018 (rust-lang/reference#752)
- Update macro-ambiguity.md (rust-lang/reference#754)
- typo fix: add missing `by` (rust-lang/reference#753)
- fix `TypeParamBounds` link on trait objects (rust-lang/reference#749)
- reorganize docs on references (rust-lang/reference#745)
- add MacroRepOp usage for ? (rust-lang/reference#744)

## book

49 commits in 87dd6843678575f8dda962f239d14ef4be14b352..6fb3705e5230311b096d47f7e2c91f9ce24393d0
2020-01-20 15:20:40 -0500 to 2020-02-12 13:48:57 -0500
- Fix nomicon links. (rust-lang/book#2253)
- Update to Rust 1.41.0 (rust-lang/book#2244)
- Listing 19-6: use ptr.add instead of ptr.offset (rust-lang/book#2201)
- Remove unneeded mutable reference
- Clarify deref coercion explanation
- Fix typo in link to 1.30 book
- Acknowledge Murphy's Law
- Clarify that buffer overread is UB in C
- Change from "must" to "idiomatic" about comments
- Fancy quotes
- Make HashMap types match previous example; add fwd ref to ch 13
- Tweak wording to array clarification
- Merge remote-tracking branch 'origin/pr/2236'
- Update all our crates (rust-lang/book#2235)
- Reword git caveat
- Merge remote-tracking branch 'origin/pr/2234'
- Merge remote-tracking branch 'origin/pr/2230'
- println! is a macro (rust-lang/book#2224)
- Update a translated version link (rust-lang/book#2221)
- move `Macro invocation` from section on tuple to section on mac… (rust-lang/book#2206)
- Do not limit `Self` usage in trait implementation (rust-lang/book#2197)
- Merge remote-tracking branch 'origin/pr/2191'
- Fix wrapping
- Merge remote-tracking branch 'origin/pr/2187'
- Updated appendix 07 to reflect deprecation of rustup install (rust-lang/book#2181)
- Make links to the Nomicon consistent
- Merge remote-tracking branch 'origin/pr/2180'
- Merge remote-tracking branch 'origin/pr/2175'
- Merge remote-tracking branch 'origin/pr/2171'
- Merge remote-tracking branch 'origin/pr/2170'
- Clarify and make consistent the explanation of unions
- Merge remote-tracking branch 'origin/pr/2166'
- Handle dev or test in the Finished output line
- Link to macros by example rather than macros (rust-lang/book#2164)
- Merge remote-tracking branch 'origin/pr/2147'
- Fix parens (rust-lang/book#2132)
- Clarify type inference with closures requires calling the closures
- Update link to French translation (rust-lang/book#2119)
- Merge remote-tracking branch 'origin/pr/2108'
- Add an explicit cross reference to data type
- Merge remote-tracking branch 'origin/pr/2105'
- ch15-02-deref: Improve explanation on immut-to-mut (rust-lang/book#2030)
- Remove unnecessary quotes
- Make markdown link identifier match
- Remove extra newline
- Merge remote-tracking branch 'origin/pr/2004'
- Extract code and output; script formatting and updating them (rust-lang/book#2231)
- Switch "Finally" to "Next" to reflect new chapters having been… (rust-lang/book#2098)
- ch19-06 added curly braces to macro output (rust-lang/book#2050)

## rust-by-example

2 commits in 1c2bd024d13f8011307e13386cf1fea2180352b5..32facd5522ddbbf37baf01e4e4b6562bc55c071a
2020-01-20 12:18:36 -0300 to 2020-02-11 09:25:06 -0300
- Add missing `dyn` in code sample (rust-lang/rust-by-example#1306)
- Improve grammar in a few sections (rust-lang/rust-by-example#1305)

## edition-guide

1 commits in 1a2390247ad6d08160e0dd74f40a01a9578659c2..37f9e6848411188a1062ead1bd8ebe4b8aa16899
2019-12-29 10:40:55 -0800 to 2020-02-10 14:36:14 +0100
- Fixed typo (rust-lang/edition-guide#196)

## embedded-book

4 commits in 4d78994915af1bde9a95c04a8c27d8dca066232a..b2e1092bf67bd4d7686c4553f186edbb7f5f92db
2020-01-14 08:25:25 +0000 to 2020-01-30 08:45:46 +0000
- Make typestate initialization notes correct  (rust-embedded/book#224)
- Mention discovery book more prominently  (rust-embedded/book#219)
- Replace nursery links with rust-lang links  (rust-embedded/book#222)
- Add a Glossary appendix page  (rust-embedded/book#223)

src/bootstrap/test.rs

@@ -1264,28 +1264,75 @@ impl Step for Compiletest {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
-struct DocTest {
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct BookTest {
     compiler: Compiler,
-    path: &'static str,
+    path: PathBuf,
     name: &'static str,
     is_ext_doc: bool,
 }
 
-impl Step for DocTest {
+impl Step for BookTest {
     type Output = ();
     const ONLY_HOSTS: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.never()
     }
 
-    /// Runs `rustdoc --test` for all documentation in `src/doc`.
+    /// Runs the documentation tests for a book in `src/doc`.
     ///
-    /// This will run all tests in our markdown documentation (e.g., the book)
-    /// located in `src/doc`. The `rustdoc` that's run is the one that sits next to
-    /// `compiler`.
+    /// This uses the `rustdoc` that sits next to `compiler`.
     fn run(self, builder: &Builder<'_>) {
+        // External docs are different from local because:
+        // - Some books need pre-processing by mdbook before being tested.
+        // - They need to save their state to toolstate.
+        // - They are only tested on the "checktools" builders.
+        //
+        // The local docs are tested by default, and we don't want to pay the
+        // cost of building mdbook, so they use `rustdoc --test` directly.
+        // Also, the unstable book is special because SUMMARY.md is generated,
+        // so it is easier to just run `rustdoc` on its files.
+        if self.is_ext_doc {
+            self.run_ext_doc(builder);
+        } else {
+            self.run_local_doc(builder);
+        }
+    }
+}
+
+impl BookTest {
+    /// This runs the equivalent of `mdbook test` (via the rustbook wrapper)
+    /// which in turn runs `rustdoc --test` on each file in the book.
+    fn run_ext_doc(self, builder: &Builder<'_>) {
+        let compiler = self.compiler;
+
+        builder.ensure(compile::Std { compiler, target: compiler.host });
+
+        // mdbook just executes a binary named "rustdoc", so we need to update
+        // PATH so that it points to our rustdoc.
+        let mut rustdoc_path = builder.rustdoc(compiler);
+        rustdoc_path.pop();
+        let old_path = env::var_os("PATH").unwrap_or_default();
+        let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path)))
+            .expect("could not add rustdoc to PATH");
+
+        let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+        let path = builder.src.join(&self.path);
+        rustbook_cmd.env("PATH", new_path).arg("test").arg(path);
+        builder.add_rust_test_threads(&mut rustbook_cmd);
+        builder.info(&format!("Testing rustbook {}", self.path.display()));
+        let _time = util::timeit(&builder);
+        let toolstate = if try_run(builder, &mut rustbook_cmd) {
+            ToolState::TestPass
+        } else {
+            ToolState::TestFail
+        };
+        builder.save_toolstate(self.name, toolstate);
+    }
+
+    /// This runs `rustdoc --test` on all `.md` files in the path.
+    fn run_local_doc(self, builder: &Builder<'_>) {
         let compiler = self.compiler;
 
         builder.ensure(compile::Std { compiler, target: compiler.host });
@@ -1294,7 +1341,6 @@ impl Step for DocTest {
         // tests for all files that end in `*.md`
         let mut stack = vec![builder.src.join(self.path)];
         let _time = util::timeit(&builder);
-
         let mut files = Vec::new();
         while let Some(p) = stack.pop() {
             if p.is_dir() {
@@ -1306,25 +1352,13 @@ impl Step for DocTest {
                 continue;
             }
 
-            // The nostarch directory in the book is for no starch, and so isn't
-            // guaranteed to builder. We don't care if it doesn't build, so skip it.
-            if p.to_str().map_or(false, |p| p.contains("nostarch")) {
-                continue;
-            }
-
             files.push(p);
         }
 
         files.sort();
 
-        let mut toolstate = ToolState::TestPass;
         for file in files {
-            if !markdown_test(builder, compiler, &file) {
-                toolstate = ToolState::TestFail;
-            }
-        }
-        if self.is_ext_doc {
-            builder.save_toolstate(self.name, toolstate);
+            markdown_test(builder, compiler, &file);
         }
     }
 }
@@ -1353,9 +1387,9 @@ macro_rules! test_book {
                 }
 
                 fn run(self, builder: &Builder<'_>) {
-                    builder.ensure(DocTest {
+                    builder.ensure(BookTest {
                         compiler: self.compiler,
-                        path: $path,
+                        path: PathBuf::from($path),
                         name: $book_name,
                         is_ext_doc: !$default,
                     });

src/doc/rustdoc/book.toml

@@ -0,0 +1,4 @@
+[book]
+authors = ["The Rust Project Developers"]
+src = "src"
+title = "The rustdoc book"

src/tools/rustbook/src/main.rs

@@ -25,6 +25,11 @@ fn main() {
                 .arg_from_usage(d_message)
                 .arg_from_usage(dir_message),
         )
+        .subcommand(
+            SubCommand::with_name("test")
+                .about("Tests that a book's Rust code samples compile")
+                .arg_from_usage(dir_message),
+        )
         .subcommand(
             SubCommand::with_name("linkcheck")
                 .about("Run linkcheck with mdBook 3")
@@ -36,13 +41,12 @@ fn main() {
     match matches.subcommand() {
         ("build", Some(sub_matches)) => {
             if let Err(e) = build(sub_matches) {
-                eprintln!("Error: {}", e);
-
-                for cause in e.iter().skip(1) {
-                    eprintln!("\tCaused By: {}", cause);
-                }
-
-                ::std::process::exit(101);
+                handle_error(e);
+            }
+        }
+        ("test", Some(sub_matches)) => {
+            if let Err(e) = test(sub_matches) {
+                handle_error(e);
             }
         }
         ("linkcheck", Some(sub_matches)) => {
@@ -148,6 +152,12 @@ pub fn build(args: &ArgMatches<'_>) -> Result3<()> {
     Ok(())
 }
 
+fn test(args: &ArgMatches<'_>) -> Result3<()> {
+    let book_dir = get_book_dir(args);
+    let mut book = MDBook::load(&book_dir)?;
+    book.test(vec![])
+}
+
 fn get_book_dir(args: &ArgMatches<'_>) -> PathBuf {
     if let Some(dir) = args.value_of("dir") {
         // Check if path is relative from current dir, or absolute...
@@ -157,3 +167,13 @@ fn get_book_dir(args: &ArgMatches<'_>) -> PathBuf {
         env::current_dir().unwrap()
     }
 }
+
+fn handle_error(error: mdbook::errors::Error) -> ! {
+    eprintln!("Error: {}", error);
+
+    for cause in error.iter().skip(1) {
+        eprintln!("\tCaused By: {}", cause);
+    }
+
+    ::std::process::exit(101);
+}

src/tools/tidy/src/lib.rs

@@ -58,6 +58,7 @@ fn filter_dirs(path: &Path) -> bool {
         "src/tools/rls",
         "src/tools/rust-installer",
         "src/tools/rustfmt",
+        "src/doc/book",
         // Filter RLS output directories
         "target/rls",
     ];