From 6ee8e0fc112fbc126405a75dff0275dd50b1dbe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Mar 2018 10:05:33 +0100 Subject: [PATCH 1/6] Add par_body_owners --- src/librustc/ty/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/librustc/ty/mod.rs b/src/librustc/ty/mod.rs index 229e5606afa96..24e81ed450f6c 100644 --- a/src/librustc/ty/mod.rs +++ b/src/librustc/ty/mod.rs @@ -44,7 +44,7 @@ use std::cmp::{self, Ordering}; use std::fmt; use std::hash::{Hash, Hasher}; use std::ops::Deref; -use rustc_data_structures::sync::Lrc; +use rustc_data_structures::sync::{self, Lrc, ParallelIterator, par_iter}; use std::slice; use std::vec::IntoIter; use std::mem; @@ -2436,6 +2436,12 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { .map(move |&body_id| self.hir.body_owner_def_id(body_id)) } + pub fn par_body_owners(self, f: F) { + par_iter(&self.hir.krate().body_ids).for_each(|&body_id| { + f(self.hir.body_owner_def_id(body_id)) + }); + } + pub fn expr_span(self, id: NodeId) -> Span { match self.hir.find(id) { Some(hir_map::NodeExpr(e)) => { From b5650f9282804f88dafa2d800ac9fae27765ddcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Mar 2018 10:08:52 +0100 Subject: [PATCH 2/6] Parallel code --- src/librustc_borrowck/borrowck/mod.rs | 4 ++-- src/librustc_borrowck/lib.rs | 2 ++ src/librustc_driver/driver.rs | 8 +++----- src/librustc_typeck/check/mod.rs | 4 ++-- src/librustc_typeck/lib.rs | 2 ++ 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/librustc_borrowck/borrowck/mod.rs b/src/librustc_borrowck/borrowck/mod.rs index 11d35def00774..684fd10c8c652 100644 --- a/src/librustc_borrowck/borrowck/mod.rs +++ b/src/librustc_borrowck/borrowck/mod.rs @@ -67,9 +67,9 @@ pub struct LoanDataFlowOperator; pub type LoanDataFlow<'a, 'tcx> = DataFlowContext<'a, 'tcx, LoanDataFlowOperator>; pub fn check_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { - for body_owner_def_id in tcx.body_owners() { + tcx.par_body_owners(|body_owner_def_id| { tcx.borrowck(body_owner_def_id); - } + }); } pub fn provide(providers: &mut Providers) { diff --git a/src/librustc_borrowck/lib.rs b/src/librustc_borrowck/lib.rs index 52a357e1a1d31..a5a20af0e4e4a 100644 --- a/src/librustc_borrowck/lib.rs +++ b/src/librustc_borrowck/lib.rs @@ -17,6 +17,8 @@ #![feature(from_ref)] #![feature(quote)] +#![recursion_limit="256"] + #[macro_use] extern crate log; extern crate syntax; extern crate syntax_pos; diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index 5d5baf765497e..c18a089268659 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -1272,11 +1272,9 @@ where time(sess, "borrow checking", || borrowck::check_crate(tcx)); - time(sess, "MIR borrow checking", || { - for def_id in tcx.body_owners() { - tcx.mir_borrowck(def_id); - } - }); + time(sess, + "MIR borrow checking", + || tcx.par_body_owners(|def_id| { tcx.mir_borrowck(def_id); })); time(sess, "dumping chalk-like clauses", || { rustc_traits::lowering::dump_program_clauses(tcx); diff --git a/src/librustc_typeck/check/mod.rs b/src/librustc_typeck/check/mod.rs index d68fc67073e9d..b70b61d191594 100644 --- a/src/librustc_typeck/check/mod.rs +++ b/src/librustc_typeck/check/mod.rs @@ -702,9 +702,9 @@ fn typeck_item_bodies<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, crate_num: CrateNum { debug_assert!(crate_num == LOCAL_CRATE); Ok(tcx.sess.track_errors(|| { - for body_owner_def_id in tcx.body_owners() { + tcx.par_body_owners(|body_owner_def_id| { ty::query::queries::typeck_tables_of::ensure(tcx, body_owner_def_id); - } + }); })?) } diff --git a/src/librustc_typeck/lib.rs b/src/librustc_typeck/lib.rs index ce7249bd7b5b7..80f57adf580bf 100644 --- a/src/librustc_typeck/lib.rs +++ b/src/librustc_typeck/lib.rs @@ -83,6 +83,8 @@ This API is completely unstable and subject to change. #![feature(slice_sort_by_cached_key)] #![feature(never_type)] +#![recursion_limit="256"] + #[macro_use] extern crate log; #[macro_use] extern crate syntax; extern crate syntax_pos; From 71c26b3171a62a0ba70bfe0f074653d236e6e20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 24 Jan 2018 09:59:08 +0100 Subject: [PATCH 3/6] Save query results and the dep graph in parallel --- src/librustc_incremental/lib.rs | 2 ++ src/librustc_incremental/persist/save.rs | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index ababce69e3170..3839c133a6eb2 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -17,6 +17,8 @@ #![feature(fs_read_write)] #![feature(specialization)] +#![recursion_limit="256"] + extern crate graphviz; #[macro_use] extern crate rustc; extern crate rustc_data_structures; diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index be725b1793321..dcef0c662c3a1 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -13,6 +13,7 @@ use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::util::common::time; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::join; use rustc_serialize::Encodable as RustcEncodable; use rustc_serialize::opaque::Encoder; use std::io::{self, Cursor}; @@ -33,23 +34,28 @@ pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { return; } - time(sess, "persist query result cache", || { - save_in(sess, - query_cache_path(sess), - |e| encode_query_cache(tcx, e)); - }); + let query_cache_path = query_cache_path(sess); + let dep_graph_path = dep_graph_path(sess); - if tcx.sess.opts.debugging_opts.incremental_queries { + join(move || { + if tcx.sess.opts.debugging_opts.incremental_queries { + time(sess, "persist query result cache", || { + save_in(sess, + query_cache_path, + |e| encode_query_cache(tcx, e)); + }); + } + }, || { time(sess, "persist dep-graph", || { save_in(sess, - dep_graph_path(sess), + dep_graph_path, |e| { time(sess, "encode dep-graph", || { encode_dep_graph(tcx, e) }) }); }); - } + }); dirty_clean::check_dirty_clean_annotations(tcx); }) From d86eb784d65781849189b6ffb0d23404b6d1241e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 21 Jan 2018 19:46:14 +0100 Subject: [PATCH 4/6] Parallelize trans item collection --- src/librustc_mir/lib.rs | 3 ++ src/librustc_mir/monomorphize/collector.rs | 42 ++++++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/librustc_mir/lib.rs b/src/librustc_mir/lib.rs index d815d4a7dba91..dc0d0b244633f 100644 --- a/src/librustc_mir/lib.rs +++ b/src/librustc_mir/lib.rs @@ -34,7 +34,10 @@ Rust MIR: a lowered representation of Rust. Also: an experiment! #![feature(specialization)] #![feature(try_trait)] +#![recursion_limit="256"] + extern crate arena; + #[macro_use] extern crate bitflags; #[macro_use] extern crate log; diff --git a/src/librustc_mir/monomorphize/collector.rs b/src/librustc_mir/monomorphize/collector.rs index 1fb1217283807..9e1223fe29cd6 100644 --- a/src/librustc_mir/monomorphize/collector.rs +++ b/src/librustc_mir/monomorphize/collector.rs @@ -207,10 +207,12 @@ use rustc::mir::interpret::{Scalar, GlobalId, AllocType}; use monomorphize::{self, Instance}; use rustc::util::nodemap::{FxHashSet, FxHashMap, DefIdMap}; +use rustc::util::common::time; use monomorphize::item::{MonoItemExt, DefPathBasedNames, InstantiationMode}; use rustc_data_structures::bitvec::BitVector; +use rustc_data_structures::sync::{ParallelIterator, par_iter, Lock}; #[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)] pub enum MonoItemCollectionMode { @@ -298,22 +300,26 @@ pub fn collect_crate_mono_items<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, mode: MonoItemCollectionMode) -> (FxHashSet>, InliningMap<'tcx>) { - let roots = collect_roots(tcx, mode); + let roots = time(tcx.sess, "collecting roots", || { + collect_roots(tcx, mode) + }); debug!("Building mono item graph, beginning at roots"); - let mut visited = FxHashSet(); - let mut recursion_depths = DefIdMap(); - let mut inlining_map = InliningMap::new(); - - for root in roots { - collect_items_rec(tcx, - root, - &mut visited, - &mut recursion_depths, - &mut inlining_map); - } + let visited = Lock::new(FxHashSet()); + let inlining_map = Lock::new(InliningMap::new()); + + time(tcx.sess, "collecting mono items", || { + par_iter(roots).for_each(|root| { + let mut recursion_depths = DefIdMap(); + collect_items_rec(tcx, + root, + &visited, + &mut recursion_depths, + &inlining_map); + }); + }); - (visited, inlining_map) + (visited.into_inner(), inlining_map.into_inner()) } // Find all non-generic items by walking the HIR. These items serve as roots to @@ -354,10 +360,10 @@ fn collect_roots<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, // Collect all monomorphized items reachable from `starting_point` fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, starting_point: MonoItem<'tcx>, - visited: &mut FxHashSet>, + visited: &Lock>>, recursion_depths: &mut DefIdMap, - inlining_map: &mut InliningMap<'tcx>) { - if !visited.insert(starting_point.clone()) { + inlining_map: &Lock>) { + if !visited.lock().insert(starting_point.clone()) { // We've been here already, no need to search again. return; } @@ -428,7 +434,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, fn record_accesses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, caller: MonoItem<'tcx>, callees: &[MonoItem<'tcx>], - inlining_map: &mut InliningMap<'tcx>) { + inlining_map: &Lock>) { let is_inlining_candidate = |mono_item: &MonoItem<'tcx>| { mono_item.instantiation_mode(tcx) == InstantiationMode::LocalCopy }; @@ -438,7 +444,7 @@ fn record_accesses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (*mono_item, is_inlining_candidate(mono_item)) }); - inlining_map.record_accesses(caller, accesses); + inlining_map.lock().record_accesses(caller, accesses); } fn check_recursion_limit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, From 8368f364e3ae1245a4f95040487d1781570f455a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 8 Jun 2018 17:48:31 +0200 Subject: [PATCH 5/6] Add MTRef and a lock_mut function to MTLock --- src/librustc_data_structures/sync.rs | 45 +++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 33f6eda2a8753..b82fe3ec60c3b 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -26,6 +26,8 @@ //! //! `MTLock` is a mutex which disappears if cfg!(parallel_queries) is false. //! +//! `MTRef` is a immutable refernce if cfg!(parallel_queries), and an mutable reference otherwise. +//! //! `rustc_erase_owner!` erases a OwningRef owner into Erased or Erased + Send + Sync //! depending on the value of cfg!(parallel_queries). @@ -126,6 +128,8 @@ cfg_if! { } } + pub type MTRef<'a, T> = &'a mut T; + #[derive(Debug)] pub struct MTLock(T); @@ -151,13 +155,8 @@ cfg_if! { } #[inline(always)] - pub fn borrow(&self) -> &T { - &self.0 - } - - #[inline(always)] - pub fn borrow_mut(&self) -> &T { - &self.0 + pub fn lock_mut(&mut self) -> &mut T { + &mut self.0 } } @@ -221,7 +220,37 @@ cfg_if! { pub use std::sync::Arc as Lrc; pub use std::sync::Weak as Weak; - pub use self::Lock as MTLock; + pub type MTRef<'a, T> = &'a T; + + #[derive(Debug)] + pub struct MTLock(Lock); + + impl MTLock { + #[inline(always)] + pub fn new(inner: T) -> Self { + MTLock(Lock::new(inner)) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } + + #[inline(always)] + pub fn lock(&self) -> LockGuard { + self.0.lock() + } + + #[inline(always)] + pub fn lock_mut(&self) -> LockGuard { + self.lock() + } + } use parking_lot::Mutex as InnerLock; use parking_lot::RwLock as InnerRwLock; From fe1cb88c08737a73e98f94d077bb23064de18106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 8 Jun 2018 17:49:21 +0200 Subject: [PATCH 6/6] Use MTRef and MTLock to avoid a RefCell --- src/librustc_mir/monomorphize/collector.rs | 42 ++++++++++++---------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/librustc_mir/monomorphize/collector.rs b/src/librustc_mir/monomorphize/collector.rs index 9e1223fe29cd6..96aeb969d89f9 100644 --- a/src/librustc_mir/monomorphize/collector.rs +++ b/src/librustc_mir/monomorphize/collector.rs @@ -212,7 +212,7 @@ use rustc::util::common::time; use monomorphize::item::{MonoItemExt, DefPathBasedNames, InstantiationMode}; use rustc_data_structures::bitvec::BitVector; -use rustc_data_structures::sync::{ParallelIterator, par_iter, Lock}; +use rustc_data_structures::sync::{MTRef, MTLock, ParallelIterator, par_iter}; #[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)] pub enum MonoItemCollectionMode { @@ -305,19 +305,25 @@ pub fn collect_crate_mono_items<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }); debug!("Building mono item graph, beginning at roots"); - let visited = Lock::new(FxHashSet()); - let inlining_map = Lock::new(InliningMap::new()); - - time(tcx.sess, "collecting mono items", || { - par_iter(roots).for_each(|root| { - let mut recursion_depths = DefIdMap(); - collect_items_rec(tcx, - root, - &visited, - &mut recursion_depths, - &inlining_map); + + let mut visited = MTLock::new(FxHashSet()); + let mut inlining_map = MTLock::new(InliningMap::new()); + + { + let visited: MTRef<'_, _> = &mut visited; + let inlining_map: MTRef<'_, _> = &mut inlining_map; + + time(tcx.sess, "collecting mono items", || { + par_iter(roots).for_each(|root| { + let mut recursion_depths = DefIdMap(); + collect_items_rec(tcx, + root, + visited, + &mut recursion_depths, + inlining_map); + }); }); - }); + } (visited.into_inner(), inlining_map.into_inner()) } @@ -360,10 +366,10 @@ fn collect_roots<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, // Collect all monomorphized items reachable from `starting_point` fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, starting_point: MonoItem<'tcx>, - visited: &Lock>>, + visited: MTRef<'_, MTLock>>>, recursion_depths: &mut DefIdMap, - inlining_map: &Lock>) { - if !visited.lock().insert(starting_point.clone()) { + inlining_map: MTRef<'_, MTLock>>) { + if !visited.lock_mut().insert(starting_point.clone()) { // We've been here already, no need to search again. return; } @@ -434,7 +440,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, fn record_accesses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, caller: MonoItem<'tcx>, callees: &[MonoItem<'tcx>], - inlining_map: &Lock>) { + inlining_map: MTRef<'_, MTLock>>) { let is_inlining_candidate = |mono_item: &MonoItem<'tcx>| { mono_item.instantiation_mode(tcx) == InstantiationMode::LocalCopy }; @@ -444,7 +450,7 @@ fn record_accesses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (*mono_item, is_inlining_candidate(mono_item)) }); - inlining_map.lock().record_accesses(caller, accesses); + inlining_map.lock_mut().record_accesses(caller, accesses); } fn check_recursion_limit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,