Auto merge of rust-lang#122070 - Zoxc:dep-edges-from-previous, r=<try>

Encode dep graph edges directly from the previous graph when promoting

This encodes dep graph edges directly from the previous graph when promoting nodes from a previous session, avoiding allocations / copies.

Based on rust-lang#122064 and rust-lang#116375.

<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:unchanged</td><td align="right">0.4177s</td><td align="right">0.4072s</td><td align="right">💚  -2.52%</td></tr><tr><td>🟣 <b>hyper</b>:check:unchanged</td><td align="right">0.1430s</td><td align="right">0.1420s</td><td align="right"> -0.69%</td></tr><tr><td>🟣 <b>regex</b>:check:unchanged</td><td align="right">0.3106s</td><td align="right">0.3038s</td><td align="right">💚  -2.19%</td></tr><tr><td>🟣 <b>syn</b>:check:unchanged</td><td align="right">0.5823s</td><td align="right">0.5688s</td><td align="right">💚  -2.33%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:unchanged</td><td align="right">1.3992s</td><td align="right">1.3692s</td><td align="right">💚  -2.14%</td></tr><tr><td>Total</td><td align="right">2.8528s</td><td align="right">2.7910s</td><td align="right">💚  -2.17%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9803s</td><td align="right">💚  -1.97%</td></tr></table>

compiler/rustc_incremental/src/persist/load.rs

@@ -11,6 +11,7 @@ use rustc_session::config::IncrementalStateAssertion;
 use rustc_session::{Session, StableCrateId};
 use rustc_span::{ErrorGuaranteed, Symbol};
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use super::data::*;
 use super::file_format;
@@ -88,7 +89,7 @@ fn delete_dirty_work_product(sess: &Session, swp: SerializedWorkProduct) {
     work_product::delete_workproduct_files(sess, &swp.work_product);
 }
 
-fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProductMap)> {
+fn load_dep_graph(sess: &Session) -> LoadResult<(Arc<SerializedDepGraph>, WorkProductMap)> {
     let prof = sess.prof.clone();
 
     if sess.opts.incremental.is_none() {
@@ -161,7 +162,7 @@ fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProduct
                 return LoadResult::DataOutOfDate;
             }
 
-            let dep_graph = SerializedDepGraph::decode::<DepsType>(&mut decoder);
+            let dep_graph = SerializedDepGraph::decode::<DepsType>(&mut decoder, sess);
 
             LoadResult::Ok { data: (dep_graph, prev_work_products) }
         }

compiler/rustc_incremental/src/persist/save.rs

@@ -10,6 +10,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_serialize::Encodable as RustcEncodable;
 use rustc_session::Session;
 use std::fs;
+use std::sync::Arc;
 
 use super::data::*;
 use super::dirty_clean;
@@ -147,7 +148,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult
 /// and moves it to the permanent dep-graph path
 pub(crate) fn build_dep_graph(
     sess: &Session,
-    prev_graph: SerializedDepGraph,
+    prev_graph: Arc<SerializedDepGraph>,
     prev_work_products: WorkProductMap,
 ) -> Option<DepGraph> {
     if sess.opts.incremental.is_none() {

compiler/rustc_middle/src/ty/context.rs

@@ -709,7 +709,7 @@ impl<'tcx> GlobalCtxt<'tcx> {
     }
 
     pub fn finish(&self) -> FileEncodeResult {
-        self.dep_graph.finish_encoding(&self.sess.prof)
+        self.dep_graph.finish_encoding()
     }
 }
 

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -1,9 +1,8 @@
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::profiling::{EventId, QueryInvocationId, SelfProfilerRef};
+use rustc_data_structures::profiling::{QueryInvocationId, SelfProfilerRef};
 use rustc_data_structures::sharded::{self, Sharded};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc};
 use rustc_data_structures::unord::UnordMap;
 use rustc_index::IndexVec;
@@ -14,6 +13,7 @@ use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
 use std::sync::atomic::Ordering;
+use std::sync::Arc;
 
 use super::query::DepGraphQuery;
 use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
@@ -82,7 +82,7 @@ pub(crate) struct DepGraphData<D: Deps> {
 
     /// The dep-graph from the previous compilation session. It contains all
     /// nodes and edges as well as all fingerprints of nodes that have them.
-    previous: SerializedDepGraph,
+    previous: Arc<SerializedDepGraph>,
 
     colors: DepNodeColorMap,
 
@@ -114,7 +114,7 @@ where
 impl<D: Deps> DepGraph<D> {
     pub fn new(
         profiler: &SelfProfilerRef,
-        prev_graph: SerializedDepGraph,
+        prev_graph: Arc<SerializedDepGraph>,
         prev_work_products: WorkProductMap,
         encoder: FileEncoder,
         record_graph: bool,
@@ -128,13 +128,13 @@ impl<D: Deps> DepGraph<D> {
             encoder,
             record_graph,
             record_stats,
+            prev_graph.clone(),
         );
 
         let colors = DepNodeColorMap::new(prev_graph_node_count);
 
         // Instantiate a dependy-less node only once for anonymous queries.
         let _green_node_index = current.intern_new_node(
-            profiler,
             DepNode { kind: D::DEP_KIND_NULL, hash: current.anon_id_seed.into() },
             EdgesVec::new(),
             Fingerprint::ZERO,
@@ -143,7 +143,6 @@ impl<D: Deps> DepGraph<D> {
 
         // Instantiate a dependy-less red node only once for anonymous queries.
         let (red_node_index, red_node_prev_index_and_color) = current.intern_node(
-            profiler,
             &prev_graph,
             DepNode { kind: D::DEP_KIND_RED, hash: Fingerprint::ZERO.into() },
             EdgesVec::new(),
@@ -196,7 +195,7 @@ impl<D: Deps> DepGraph<D> {
 
     pub fn with_query(&self, f: impl Fn(&DepGraphQuery)) {
         if let Some(data) = &self.data {
-            data.current.encoder.borrow().with_query(f)
+            data.current.encoder.with_query(f)
         }
     }
 
@@ -372,13 +371,8 @@ impl<D: Deps> DepGraphData<D> {
             hash_result.map(|f| dcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, &result)));
 
         // Intern the new `DepNode`.
-        let (dep_node_index, prev_and_color) = self.current.intern_node(
-            dcx.profiler(),
-            &self.previous,
-            key,
-            edges,
-            current_fingerprint,
-        );
+        let (dep_node_index, prev_and_color) =
+            self.current.intern_node(&self.previous, key, edges, current_fingerprint);
 
         hashing_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -443,12 +437,7 @@ impl<D: Deps> DepGraphData<D> {
                     hash: self.current.anon_id_seed.combine(hasher.finish()).into(),
                 };
 
-                self.current.intern_new_node(
-                    cx.profiler(),
-                    target_dep_node,
-                    task_deps,
-                    Fingerprint::ZERO,
-                )
+                self.current.intern_new_node(target_dep_node, task_deps, Fingerprint::ZERO)
             }
         };
 
@@ -585,13 +574,8 @@ impl<D: Deps> DepGraph<D> {
             });
 
             // Intern the new `DepNode` with the dependencies up-to-now.
-            let (dep_node_index, prev_and_color) = data.current.intern_node(
-                cx.profiler(),
-                &data.previous,
-                node,
-                edges,
-                current_fingerprint,
-            );
+            let (dep_node_index, prev_and_color) =
+                data.current.intern_node(&data.previous, node, edges, current_fingerprint);
 
             hashing_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -871,11 +855,8 @@ impl<D: Deps> DepGraphData<D> {
 
         // We allocating an entry for the node in the current dependency graph and
         // adding all the appropriate edges imported from the previous graph
-        let dep_node_index = self.current.promote_node_and_deps_to_current(
-            qcx.dep_context().profiler(),
-            &self.previous,
-            prev_dep_node_index,
-        );
+        let dep_node_index =
+            self.current.promote_node_and_deps_to_current(&self.previous, prev_dep_node_index);
 
         // ... emitting any stored diagnostic ...
 
@@ -974,19 +955,15 @@ impl<D: Deps> DepGraph<D> {
 
     pub fn print_incremental_info(&self) {
         if let Some(data) = &self.data {
-            data.current.encoder.borrow().print_incremental_info(
+            data.current.encoder.print_incremental_info(
                 data.current.total_read_count.load(Ordering::Relaxed),
                 data.current.total_duplicate_read_count.load(Ordering::Relaxed),
             )
         }
     }
 
-    pub fn finish_encoding(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
-        if let Some(data) = &self.data {
-            data.current.encoder.steal().finish(profiler)
-        } else {
-            Ok(0)
-        }
+    pub fn finish_encoding(&self) -> FileEncodeResult {
+        if let Some(data) = &self.data { data.current.encoder.finish() } else { Ok(0) }
     }
 
     pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex {
@@ -1069,7 +1046,7 @@ rustc_index::newtype_index! {
 /// manipulating both, we acquire `new_node_to_index` or `prev_index_to_index`
 /// first, and `data` second.
 pub(super) struct CurrentDepGraph<D: Deps> {
-    encoder: Steal<GraphEncoder<D>>,
+    encoder: GraphEncoder<D>,
     new_node_to_index: Sharded<FxHashMap<DepNode, DepNodeIndex>>,
     prev_index_to_index: Lock<IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>>,
 
@@ -1100,12 +1077,6 @@ pub(super) struct CurrentDepGraph<D: Deps> {
     /// debugging and only active with `debug_assertions`.
     total_read_count: AtomicU64,
     total_duplicate_read_count: AtomicU64,
-
-    /// The cached event id for profiling node interning. This saves us
-    /// from having to look up the event id every time we intern a node
-    /// which may incur too much overhead.
-    /// This will be None if self-profiling is disabled.
-    node_intern_event_id: Option<EventId>,
 }
 
 impl<D: Deps> CurrentDepGraph<D> {
@@ -1115,6 +1086,7 @@ impl<D: Deps> CurrentDepGraph<D> {
         encoder: FileEncoder,
         record_graph: bool,
         record_stats: bool,
+        previous: Arc<SerializedDepGraph>,
     ) -> Self {
         use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -1140,17 +1112,15 @@ impl<D: Deps> CurrentDepGraph<D> {
 
         let new_node_count_estimate = 102 * prev_graph_node_count / 100 + 200;
 
-        let node_intern_event_id = profiler
-            .get_or_alloc_cached_string("incr_comp_intern_dep_graph_node")
-            .map(EventId::from_label);
-
         CurrentDepGraph {
-            encoder: Steal::new(GraphEncoder::new(
+            encoder: GraphEncoder::new(
                 encoder,
                 prev_graph_node_count,
                 record_graph,
                 record_stats,
-            )),
+                profiler,
+                previous,
+            ),
             new_node_to_index: Sharded::new(|| {
                 FxHashMap::with_capacity_and_hasher(
                     new_node_count_estimate / sharded::shards(),
@@ -1165,7 +1135,6 @@ impl<D: Deps> CurrentDepGraph<D> {
             fingerprints: Lock::new(IndexVec::from_elem_n(None, new_node_count_estimate)),
             total_read_count: AtomicU64::new(0),
             total_duplicate_read_count: AtomicU64::new(0),
-            node_intern_event_id,
         }
     }
 
@@ -1183,16 +1152,14 @@ impl<D: Deps> CurrentDepGraph<D> {
     #[inline(always)]
     fn intern_new_node(
         &self,
-        profiler: &SelfProfilerRef,
         key: DepNode,
         edges: EdgesVec,
         current_fingerprint: Fingerprint,
     ) -> DepNodeIndex {
         let dep_node_index = match self.new_node_to_index.lock_shard_by_value(&key).entry(key) {
             Entry::Occupied(entry) => *entry.get(),
             Entry::Vacant(entry) => {
-                let dep_node_index =
-                    self.encoder.borrow().send(profiler, key, current_fingerprint, edges);
+                let dep_node_index = self.encoder.send(key, current_fingerprint, edges);
                 entry.insert(dep_node_index);
                 dep_node_index
             }
@@ -1206,25 +1173,19 @@ impl<D: Deps> CurrentDepGraph<D> {
 
     fn intern_node(
         &self,
-        profiler: &SelfProfilerRef,
         prev_graph: &SerializedDepGraph,
         key: DepNode,
         edges: EdgesVec,
         fingerprint: Option<Fingerprint>,
     ) -> (DepNodeIndex, Option<(SerializedDepNodeIndex, DepNodeColor)>) {
-        // Get timer for profiling `DepNode` interning
-        let _node_intern_timer =
-            self.node_intern_event_id.map(|eid| profiler.generic_activity_with_event_id(eid));
-
         if let Some(prev_index) = prev_graph.node_to_index_opt(&key) {
             let get_dep_node_index = |fingerprint| {
                 let mut prev_index_to_index = self.prev_index_to_index.lock();
 
                 let dep_node_index = match prev_index_to_index[prev_index] {
                     Some(dep_node_index) => dep_node_index,
                     None => {
-                        let dep_node_index =
-                            self.encoder.borrow().send(profiler, key, fingerprint, edges);
+                        let dep_node_index = self.encoder.send(key, fingerprint, edges);
                         prev_index_to_index[prev_index] = Some(dep_node_index);
                         dep_node_index
                     }
@@ -1261,15 +1222,14 @@ impl<D: Deps> CurrentDepGraph<D> {
             let fingerprint = fingerprint.unwrap_or(Fingerprint::ZERO);
 
             // This is a new node: it didn't exist in the previous compilation session.
-            let dep_node_index = self.intern_new_node(profiler, key, edges, fingerprint);
+            let dep_node_index = self.intern_new_node(key, edges, fingerprint);
 
             (dep_node_index, None)
         }
     }
 
     fn promote_node_and_deps_to_current(
         &self,
-        profiler: &SelfProfilerRef,
         prev_graph: &SerializedDepGraph,
         prev_index: SerializedDepNodeIndex,
     ) -> DepNodeIndex {
@@ -1280,16 +1240,14 @@ impl<D: Deps> CurrentDepGraph<D> {
         match prev_index_to_index[prev_index] {
             Some(dep_node_index) => dep_node_index,
             None => {
-                let key = prev_graph.index_to_node(prev_index);
-                let edges = prev_graph
-                    .edge_targets_from(prev_index)
-                    .map(|i| prev_index_to_index[i].unwrap())
-                    .collect();
-                let fingerprint = prev_graph.fingerprint_by_index(prev_index);
-                let dep_node_index = self.encoder.borrow().send(profiler, key, fingerprint, edges);
+                let dep_node_index = self.encoder.promote(prev_index, &mut *prev_index_to_index);
                 prev_index_to_index[prev_index] = Some(dep_node_index);
                 #[cfg(debug_assertions)]
-                self.record_edge(dep_node_index, key, fingerprint);
+                self.record_edge(
+                    dep_node_index,
+                    prev_graph.index_to_node(prev_index),
+                    prev_graph.fingerprint_by_index(prev_index),
+                );
                 dep_node_index
             }
         }