Implement lazy decoding of DefPathTable during incremental compilation

compiler/rustc_incremental/src/persist/load.rs

@@ -1,6 +1,7 @@
 //! Code to save/load the dep-graph from files.
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::definitions::Definitions;
 use rustc_middle::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::query::OnDiskCache;
 use rustc_middle::ty::TyCtxt;
@@ -204,7 +205,10 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
 /// If we are not in incremental compilation mode, returns `None`.
 /// Otherwise, tries to load the query result cache from disk,
 /// creating an empty cache if it could not be loaded.
-pub fn load_query_result_cache(sess: &Session) -> Option<OnDiskCache<'_>> {
+pub fn load_query_result_cache<'a>(
+    sess: &'a Session,
+    definitions: &Definitions,
+) -> Option<OnDiskCache<'a>> {
     if sess.opts.incremental.is_none() {
         return None;
     }
@@ -217,7 +221,7 @@ pub fn load_query_result_cache(sess: &Session) -> Option<OnDiskCache<'_>> {
         sess.is_nightly_build(),
     ) {
         LoadResult::Ok { data: (bytes, start_pos) } => {
-            Some(OnDiskCache::new(sess, bytes, start_pos))
+            Some(OnDiskCache::new(sess, bytes, start_pos, definitions))
         }
         _ => Some(OnDiskCache::new_empty(sess.source_map())),
     }

compiler/rustc_interface/src/passes.rs

@@ -747,7 +747,7 @@ pub fn create_global_ctxt<'tcx>(
         Definitions::new(crate_name, sess.local_crate_disambiguator()),
     ));
 
-    let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess);
+    let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess, defs);
 
     let codegen_backend = compiler.codegen_backend();
     let mut local_providers = *DEFAULT_QUERY_PROVIDERS;

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -77,6 +77,10 @@ crate struct CrateMetadata {
     raw_proc_macros: Option<&'static [ProcMacro]>,
     /// Source maps for code from the crate.
     source_map_import_info: OnceCell<Vec<ImportedSourceFile>>,
+    /// For every definition in this crate, maps its `DefPathHash` to its
+    /// `DefIndex`. See `raw_def_id_to_def_id` for more details about how
+    /// this is used.
+    def_path_hash_map: OnceCell<FxHashMap<DefPathHash, DefIndex>>,
     /// Used for decoding interpret::AllocIds in a cached & thread-safe manner.
     alloc_decoding_state: AllocDecodingState,
     /// The `DepNodeIndex` of the `DepNode` representing this upstream crate.
@@ -1556,6 +1560,53 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         })
     }
 
+    /// Finds the corresponding `DefId` for the provided `DefPathHash`, if it exists.
+    /// This is used by incremental compilation to map a serialized `DefPathHash` to
+    /// its `DefId` in the current session.
+    /// Normally, only one 'main' crate will change between incremental compilation sessions:
+    /// all dependencies will be completely unchanged. In this case, we can avoid
+    /// decoding every `DefPathHash` in the crate, since the `DefIndex` from the previous
+    /// session will still be valid. If our 'guess' is wrong (the `DefIndex` no longer exists,
+    /// or has a different `DefPathHash`, then we need to decode all `DefPathHashes` to determine
+    /// the correct mapping).
+    fn def_path_hash_to_def_id(
+        &self,
+        krate: CrateNum,
+        index_guess: u32,
+        hash: DefPathHash,
+    ) -> Option<DefId> {
+        let def_index_guess = DefIndex::from_u32(index_guess);
+        let old_hash = self
+            .root
+            .tables
+            .def_path_hashes
+            .get(self, def_index_guess)
+            .map(|lazy| lazy.decode(self));
+
+        // Fast path: the definition and its index is unchanged from the
+        // previous compilation session. There is no need to decode anything
+        // else
+        if old_hash == Some(hash) {
+            return Some(DefId { krate, index: def_index_guess });
+        }
+
+        // Slow path: We need to find out the new `DefIndex` of the provided
+        // `DefPathHash`, if its still exists. This requires decoding every `DefPathHash`
+        // stored in this crate.
+        let map = self.cdata.def_path_hash_map.get_or_init(|| {
+            let end_id = self.root.tables.def_path_hashes.size() as u32;
+            let mut map = FxHashMap::with_capacity_and_hasher(end_id as usize, Default::default());
+            for i in 0..end_id {
+                let def_index = DefIndex::from_u32(i);
+                let hash =
+                    self.root.tables.def_path_hashes.get(self, def_index).unwrap().decode(self);
+                map.insert(hash, def_index);
+            }
+            map
+        });
+        map.get(&hash).map(|index| DefId { krate, index: *index })
+    }
+
     // Returns the path leading to the thing with this `id`.
     fn def_path(&self, id: DefIndex) -> DefPath {
         debug!("def_path(cnum={:?}, id={:?})", self.cnum, id);
@@ -1834,6 +1885,7 @@ impl CrateMetadata {
             trait_impls,
             raw_proc_macros,
             source_map_import_info: OnceCell::new(),
+            def_path_hash_map: Default::default(),
             alloc_decoding_state,
             dep_node_index: AtomicCell::new(DepNodeIndex::INVALID),
             cnum,

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -506,6 +506,16 @@ impl CrateStore for CStore {
         self.get_crate_data(cnum).num_def_ids()
     }
 
+    // See `CrateMetadataRef::def_path_hash_to_def_id` for more details
+    fn def_path_hash_to_def_id(
+        &self,
+        cnum: CrateNum,
+        index_guess: u32,
+        hash: DefPathHash,
+    ) -> Option<DefId> {
+        self.get_crate_data(cnum).def_path_hash_to_def_id(cnum, index_guess, hash)
+    }
+
     fn crates_untracked(&self) -> Vec<CrateNum> {
         let mut result = vec![];
         self.iter_crate_data(|cnum, _| result.push(cnum));

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -252,8 +252,7 @@ macro_rules! define_dep_nodes {
             /// has been removed.
             fn extract_def_id(&self, tcx: TyCtxt<'tcx>) -> Option<DefId> {
                 if self.kind.can_reconstruct_query_key() {
-                    let def_path_hash = DefPathHash(self.hash.into());
-                    tcx.def_path_hash_to_def_id.as_ref()?.get(&def_path_hash).cloned()
+                    tcx.queries.on_disk_cache.as_ref()?.def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into()))
                 } else {
                     None
                 }
@@ -320,7 +319,17 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for DefId {
     }
 
     fn to_fingerprint(&self, tcx: TyCtxt<'tcx>) -> Fingerprint {
-        tcx.def_path_hash(*self).0
+        let hash = tcx.def_path_hash(*self);
+        // If this is a foreign `DefId`, store its current value
+        // in the incremental cache. When we decode the cache,
+        // we will use the old DefIndex as an initial guess for
+        // a lookup into the crate metadata.
+        if !self.is_local() {
+            if let Some(cache) = &tcx.queries.on_disk_cache {
+                cache.store_foreign_def_id_hash(*self, hash);
+            }
+        }
+        hash.0
     }
 
     fn to_debug_str(&self, tcx: TyCtxt<'tcx>) -> String {
@@ -359,7 +368,7 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for CrateNum {
 
     fn to_fingerprint(&self, tcx: TyCtxt<'tcx>) -> Fingerprint {
         let def_id = DefId { krate: *self, index: CRATE_DEF_INDEX };
-        tcx.def_path_hash(def_id).0
+        def_id.to_fingerprint(tcx)
     }
 
     fn to_debug_str(&self, tcx: TyCtxt<'tcx>) -> String {

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -5,7 +5,7 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::Diagnostic;
-use rustc_hir::def_id::LocalDefId;
+use rustc_hir::def_id::{DefPathHash, LocalDefId};
 
 mod dep_node;
 
@@ -91,6 +91,12 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     type DepKind = DepKind;
     type StableHashingContext = StableHashingContext<'tcx>;
 
+    fn register_reused_dep_path_hash(&self, hash: DefPathHash) {
+        if let Some(cache) = self.queries.on_disk_cache.as_ref() {
+            cache.register_reused_dep_path_hash(hash)
+        }
+    }
+
     fn create_stable_hashing_context(&self) -> Self::StableHashingContext {
         TyCtxt::create_stable_hashing_context(*self)
     }

compiler/rustc_middle/src/middle/cstore.rs

@@ -191,6 +191,12 @@ pub trait CrateStore {
     fn def_path_hash(&self, def: DefId) -> DefPathHash;
     fn all_def_path_hashes_and_def_ids(&self, cnum: CrateNum) -> Vec<(DefPathHash, DefId)>;
     fn num_def_ids(&self, cnum: CrateNum) -> usize;
+    fn def_path_hash_to_def_id(
+        &self,
+        cnum: CrateNum,
+        index_guess: u32,
+        hash: DefPathHash,
+    ) -> Option<DefId>;
 
     // "queries" used in resolve that aren't tracked for incremental compilation
     fn crate_name_untracked(&self, cnum: CrateNum) -> Symbol;

compiler/rustc_middle/src/ty/context.rs

@@ -1,7 +1,7 @@
 //! Type context book-keeping.
 
 use crate::arena::Arena;
-use crate::dep_graph::{self, DepConstructor, DepGraph};
+use crate::dep_graph::{self, DepGraph, DepKind, DepNode, DepNodeExt};
 use crate::hir::exports::ExportMap;
 use crate::ich::{NodeIdHashingMode, StableHashingContext};
 use crate::infer::canonical::{Canonical, CanonicalVarInfo, CanonicalVarInfos};
@@ -34,12 +34,12 @@ use rustc_data_structures::stable_hasher::{
 };
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{self, Lock, Lrc, WorkerLocal};
-use rustc_data_structures::unhash::UnhashMap;
 use rustc_errors::ErrorReported;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE};
-use rustc_hir::definitions::{DefPathHash, Definitions};
+use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId};
+use rustc_hir::def_id::{CRATE_DEF_INDEX, LOCAL_CRATE};
+use rustc_hir::definitions::Definitions;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{
@@ -945,10 +945,6 @@ pub struct GlobalCtxt<'tcx> {
     pub(crate) untracked_crate: &'tcx hir::Crate<'tcx>,
     pub(crate) definitions: &'tcx Definitions,
 
-    /// A map from `DefPathHash` -> `DefId`. Includes `DefId`s from the local crate
-    /// as well as all upstream crates. Only populated in incremental mode.
-    pub def_path_hash_to_def_id: Option<UnhashMap<DefPathHash, DefId>>,
-
     pub queries: query::Queries<'tcx>,
 
     maybe_unused_trait_imports: FxHashSet<LocalDefId>,
@@ -1113,21 +1109,6 @@ impl<'tcx> TyCtxt<'tcx> {
         let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1);
         providers[LOCAL_CRATE] = local_providers;
 
-        let def_path_hash_to_def_id = if s.opts.build_dep_graph() {
-            let capacity = definitions.def_path_table().num_def_ids()
-                + crates.iter().map(|cnum| cstore.num_def_ids(*cnum)).sum::<usize>();
-            let mut map = UnhashMap::with_capacity_and_hasher(capacity, Default::default());
-
-            map.extend(definitions.def_path_table().all_def_path_hashes_and_def_ids(LOCAL_CRATE));
-            for cnum in &crates {
-                map.extend(cstore.all_def_path_hashes_and_def_ids(*cnum).into_iter());
-            }
-
-            Some(map)
-        } else {
-            None
-        };
-
         let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default();
         for (hir_id, v) in krate.trait_map.iter() {
             let map = trait_map.entry(hir_id.owner).or_default();
@@ -1155,7 +1136,6 @@ impl<'tcx> TyCtxt<'tcx> {
             extern_prelude: resolutions.extern_prelude,
             untracked_crate: krate,
             definitions,
-            def_path_hash_to_def_id,
             queries: query::Queries::new(providers, extern_providers, on_disk_query_result_cache),
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),
@@ -1329,7 +1309,8 @@ impl<'tcx> TyCtxt<'tcx> {
         // We cannot use the query versions of crates() and crate_hash(), since
         // those would need the DepNodes that we are allocating here.
         for cnum in self.cstore.crates_untracked() {
-            let dep_node = DepConstructor::CrateMetadata(self, cnum);
+            let def_path_hash = self.def_path_hash(DefId { krate: cnum, index: CRATE_DEF_INDEX });
+            let dep_node = DepNode::from_def_path_hash(def_path_hash, DepKind::CrateMetadata);
             let crate_hash = self.cstore.crate_hash_untracked(cnum);
             self.dep_graph.with_task(
                 dep_node,