Rollup merge of rust-lang#91313 - petrochenkov:cratexp, r=Aaron1011

expand: Turn `ast::Crate` into a first class expansion target

And stop creating a fake `mod` item for the crate root when expanding a crate, thus addressing FIXMEs left in rust-lang#82238, and making a step towards a proper support for crate-level macro attributes (cc rust-lang#54726).

I haven't added token collection support for the whole crate in this PR, maybe later.
r? `@Aaron1011`

compiler/rustc_ast/src/ast.rs

@@ -517,6 +517,8 @@ pub struct Crate {
     pub attrs: Vec<Attribute>,
     pub items: Vec<P<Item>>,
     pub span: Span,
+    // Placeholder ID if the crate node is a macro placeholder.
+    pub is_placeholder: Option<NodeId>,
 }
 
 /// Possible values inside of compile-time attribute lists.

compiler/rustc_ast/src/ast_like.rs

@@ -1,7 +1,7 @@
 use super::ptr::P;
 use super::token::Nonterminal;
 use super::tokenstream::LazyTokenStream;
-use super::{Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
+use super::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
 use super::{AssocItem, Expr, ForeignItem, Item, Local, MacCallStmt};
 use super::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
 use super::{AttrVec, Attribute, Stmt, StmtKind};
@@ -276,7 +276,7 @@ derive_has_tokens_and_attrs! {
 // These ast nodes only support inert attributes, so they don't
 // store tokens (since nothing can observe them)
 derive_has_attrs_no_tokens! {
-    FieldDef, Arm, ExprField, PatField, Variant, Param, GenericParam
+    FieldDef, Arm, ExprField, PatField, Variant, Param, GenericParam, Crate
 }
 
 // These AST nodes don't support attributes, but can

compiler/rustc_ast/src/mut_visit.rs

@@ -284,6 +284,10 @@ pub trait MutVisitor: Sized {
 /// Use a map-style function (`FnOnce(T) -> T`) to overwrite a `&mut T`. Useful
 /// when using a `flat_map_*` or `filter_map_*` method within a `visit_`
 /// method. Abort the program if the closure panics.
+///
+/// FIXME: Abort on panic means that any fatal error inside `visit_clobber` will abort the compiler.
+/// Instead of aborting on catching a panic we need to reset the visited node to some valid but
+/// possibly meaningless value and rethrow the panic.
 //
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 pub fn visit_clobber<T, F>(t: &mut T, f: F)
@@ -1105,36 +1109,11 @@ pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
     visit_unsafety(unsafety, vis);
 }
 
-// FIXME: Avoid visiting the crate as a `Mod` item, flat map only the inner items if possible,
-// or make crate visiting first class if necessary.
 pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) {
-    visit_clobber(krate, |Crate { attrs, items, span }| {
-        let item_vis =
-            Visibility { kind: VisibilityKind::Public, span: span.shrink_to_lo(), tokens: None };
-        let item = P(Item {
-            ident: Ident::empty(),
-            attrs,
-            id: DUMMY_NODE_ID,
-            vis: item_vis,
-            span,
-            kind: ItemKind::Mod(Unsafe::No, ModKind::Loaded(items, Inline::Yes, span)),
-            tokens: None,
-        });
-        let items = vis.flat_map_item(item);
-
-        let len = items.len();
-        if len == 0 {
-            Crate { attrs: vec![], items: vec![], span }
-        } else if len == 1 {
-            let Item { attrs, span, kind, .. } = items.into_iter().next().unwrap().into_inner();
-            match kind {
-                ItemKind::Mod(_, ModKind::Loaded(items, ..)) => Crate { attrs, items, span },
-                _ => panic!("visitor converted a module to not a module"),
-            }
-        } else {
-            panic!("a crate cannot expand to more than one item");
-        }
-    });
+    let Crate { attrs, items, span, is_placeholder: _ } = krate;
+    visit_attrs(attrs, vis);
+    items.flat_map_in_place(|item| vis.flat_map_item(item));
+    vis.visit_span(span);
 }
 
 // Mutates one item into possibly many items.

compiler/rustc_ast/src/visit.rs

@@ -211,6 +211,9 @@ pub trait Visitor<'ast>: Sized {
     fn visit_pat_field(&mut self, fp: &'ast PatField) {
         walk_pat_field(self, fp)
     }
+    fn visit_crate(&mut self, krate: &'ast Crate) {
+        walk_crate(self, krate)
+    }
 }
 
 #[macro_export]

compiler/rustc_ast_pretty/src/pprust/mod.rs

@@ -75,3 +75,12 @@ pub fn attribute_to_string(attr: &ast::Attribute) -> String {
 pub fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
     State::new().to_string(f)
 }
+
+pub fn crate_to_string_for_macros(krate: &ast::Crate) -> String {
+    State::new().to_string(|s| {
+        s.print_inner_attributes(&krate.attrs);
+        for item in &krate.items {
+            s.print_item(item);
+        }
+    })
+}

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -77,6 +77,10 @@ fn flat_map_annotatable(
         Annotatable::Param(param) => vis.flat_map_param(param).pop().map(Annotatable::Param),
         Annotatable::FieldDef(sf) => vis.flat_map_field_def(sf).pop().map(Annotatable::FieldDef),
         Annotatable::Variant(v) => vis.flat_map_variant(v).pop().map(Annotatable::Variant),
+        Annotatable::Crate(mut krate) => {
+            vis.visit_crate(&mut krate);
+            Some(Annotatable::Crate(krate))
+        }
     }
 }
 
@@ -101,6 +105,7 @@ impl CfgFinder {
             Annotatable::Param(param) => finder.visit_param(&param),
             Annotatable::FieldDef(field) => finder.visit_field_def(&field),
             Annotatable::Variant(variant) => finder.visit_variant(&variant),
+            Annotatable::Crate(krate) => finder.visit_crate(krate),
         };
         finder.has_cfg_or_cfg_attr
     }

compiler/rustc_builtin_macros/src/test_harness.rs

@@ -84,9 +84,35 @@ struct TestHarnessGenerator<'a> {
     tests: Vec<Test>,
 }
 
+impl TestHarnessGenerator<'_> {
+    fn add_test_cases(&mut self, node_id: ast::NodeId, span: Span, prev_tests: Vec<Test>) {
+        let mut tests = mem::replace(&mut self.tests, prev_tests);
+
+        if !tests.is_empty() {
+            // Create an identifier that will hygienically resolve the test
+            // case name, even in another module.
+            let expn_id = self.cx.ext_cx.resolver.expansion_for_ast_pass(
+                span,
+                AstPass::TestHarness,
+                &[],
+                Some(node_id),
+            );
+            for test in &mut tests {
+                // See the comment on `mk_main` for why we're using
+                // `apply_mark` directly.
+                test.ident.span =
+                    test.ident.span.apply_mark(expn_id.to_expn_id(), Transparency::Opaque);
+            }
+            self.cx.test_cases.extend(tests);
+        }
+    }
+}
+
 impl<'a> MutVisitor for TestHarnessGenerator<'a> {
     fn visit_crate(&mut self, c: &mut ast::Crate) {
+        let prev_tests = mem::take(&mut self.tests);
         noop_visit_crate(c, self);
+        self.add_test_cases(ast::CRATE_NODE_ID, c.span, prev_tests);
 
         // Create a main function to run our tests
         c.items.push(mk_main(&mut self.cx));
@@ -103,34 +129,10 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> {
 
         // We don't want to recurse into anything other than mods, since
         // mods or tests inside of functions will break things
-        if let ast::ItemKind::Mod(..) = item.kind {
-            let tests = mem::take(&mut self.tests);
+        if let ast::ItemKind::Mod(_, ModKind::Loaded(.., span)) = item.kind {
+            let prev_tests = mem::take(&mut self.tests);
             noop_visit_item_kind(&mut item.kind, self);
-            let mut tests = mem::replace(&mut self.tests, tests);
-
-            if !tests.is_empty() {
-                let parent =
-                    if item.id == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { item.id };
-                // Create an identifier that will hygienically resolve the test
-                // case name, even in another module.
-                let inner_span = match item.kind {
-                    ast::ItemKind::Mod(_, ModKind::Loaded(.., span)) => span,
-                    _ => unreachable!(),
-                };
-                let expn_id = self.cx.ext_cx.resolver.expansion_for_ast_pass(
-                    inner_span,
-                    AstPass::TestHarness,
-                    &[],
-                    Some(parent),
-                );
-                for test in &mut tests {
-                    // See the comment on `mk_main` for why we're using
-                    // `apply_mark` directly.
-                    test.ident.span =
-                        test.ident.span.apply_mark(expn_id.to_expn_id(), Transparency::Opaque);
-                }
-                self.cx.test_cases.extend(tests);
-            }
+            self.add_test_cases(item.id, span, prev_tests);
         }
         smallvec![P(item)]
     }
@@ -146,7 +148,7 @@ fn entry_point_type(sess: &Session, item: &ast::Item, depth: usize) -> EntryPoin
             } else if sess.contains_name(&item.attrs, sym::rustc_main) {
                 EntryPointType::MainAttr
             } else if item.ident.name == sym::main {
-                if depth == 1 {
+                if depth == 0 {
                     // This is a top-level function so can be 'main'
                     EntryPointType::MainNamed
                 } else {

compiler/rustc_expand/src/base.rs

@@ -48,6 +48,7 @@ pub enum Annotatable {
     Param(ast::Param),
     FieldDef(ast::FieldDef),
     Variant(ast::Variant),
+    Crate(ast::Crate),
 }
 
 impl Annotatable {
@@ -66,6 +67,7 @@ impl Annotatable {
             Annotatable::Param(ref p) => p.span,
             Annotatable::FieldDef(ref sf) => sf.span,
             Annotatable::Variant(ref v) => v.span,
+            Annotatable::Crate(ref c) => c.span,
         }
     }
 
@@ -84,6 +86,7 @@ impl Annotatable {
             Annotatable::Param(p) => p.visit_attrs(f),
             Annotatable::FieldDef(sf) => sf.visit_attrs(f),
             Annotatable::Variant(v) => v.visit_attrs(f),
+            Annotatable::Crate(c) => c.visit_attrs(f),
         }
     }
 
@@ -102,6 +105,7 @@ impl Annotatable {
             Annotatable::Param(p) => visitor.visit_param(p),
             Annotatable::FieldDef(sf) => visitor.visit_field_def(sf),
             Annotatable::Variant(v) => visitor.visit_variant(v),
+            Annotatable::Crate(c) => visitor.visit_crate(c),
         }
     }
 
@@ -122,7 +126,8 @@ impl Annotatable {
             | Annotatable::GenericParam(..)
             | Annotatable::Param(..)
             | Annotatable::FieldDef(..)
-            | Annotatable::Variant(..) => panic!("unexpected annotatable"),
+            | Annotatable::Variant(..)
+            | Annotatable::Crate(..) => panic!("unexpected annotatable"),
         }
     }
 
@@ -220,6 +225,13 @@ impl Annotatable {
             _ => panic!("expected variant"),
         }
     }
+
+    pub fn expect_crate(self) -> ast::Crate {
+        match self {
+            Annotatable::Crate(krate) => krate,
+            _ => panic!("expected krate"),
+        }
+    }
 }
 
 /// Result of an expansion that may need to be retried.
@@ -419,6 +431,11 @@ pub trait MacResult {
     fn make_variants(self: Box<Self>) -> Option<SmallVec<[ast::Variant; 1]>> {
         None
     }
+
+    fn make_crate(self: Box<Self>) -> Option<ast::Crate> {
+        // Fn-like macros cannot produce a crate.
+        unreachable!()
+    }
 }
 
 macro_rules! make_MacEager {