From 1c224f346d66098db4783b329480e948b0470f5a Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Thu, 11 Jul 2024 13:23:34 +1000 Subject: [PATCH 01/21] get and insert ty_modules --- sway-core/src/query_engine/mod.rs | 21 +++++++++++++- sway-core/src/semantic_analysis/module.rs | 34 +++++++++++++++++------ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/sway-core/src/query_engine/mod.rs b/sway-core/src/query_engine/mod.rs index 8f7e248291f..bace080fe7f 100644 --- a/sway-core/src/query_engine/mod.rs +++ b/sway-core/src/query_engine/mod.rs @@ -5,7 +5,7 @@ use sway_error::warning::CompileWarning; use sway_types::IdentUnique; use crate::decl_engine::{DeclId, DeclRef}; -use crate::language::ty::{TyFunctionDecl, TyFunctionSig}; +use crate::language::ty::{TyFunctionDecl, TyFunctionSig, TyModule}; use crate::{Engines, Programs}; pub type ModulePath = Arc; @@ -46,6 +46,13 @@ pub struct ProgramsCacheEntry { pub type ProgramsCacheMap = HashMap, ProgramsCacheEntry>; +#[derive(Clone, Debug)] +pub struct TyModuleCacheEntry { + pub path: Arc, + pub module: TyModule, +} +pub type TyModuleCacheMap = HashMap, TyModuleCacheEntry>; + #[derive(Clone, Debug)] pub struct FunctionCacheEntry { pub fn_decl: DeclRef>, @@ -59,6 +66,8 @@ pub struct QueryEngine { parse_module_cache: Arc>, programs_cache: Arc>, function_cache: Arc>, + + ty_module_cache: Arc>, } impl QueryEngine { @@ -85,6 +94,16 @@ impl QueryEngine { cache.insert(entry.path.clone(), entry); } + pub fn get_ty_module_cache_entry(&self, path: &PathBuf) -> Option { + let cache = self.ty_module_cache.read(); + cache.get(path).cloned() + } + + pub fn insert_ty_module_cache_entry(&self, entry: TyModuleCacheEntry) { + let mut cache = self.ty_module_cache.write(); + cache.insert(entry.path.clone(), entry); + } + pub fn get_function( &self, engines: &Engines, diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 2edc16ae3b7..d4cc7585d5e 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -1,9 +1,10 @@ use std::{ collections::{HashMap, HashSet}, fmt::Display, - fs, + fs, sync::Arc, }; +use fuel_vm::fuel_crypto::coins_bip32::path; use graph_cycles::Cycles; use sway_error::{ error::CompileError, @@ -12,15 +13,11 @@ use sway_error::{ use sway_types::{BaseIdent, Named}; use crate::{ - decl_engine::{DeclEngineGet, DeclId}, - engine_threading::DebugWithEngines, - language::{ + decl_engine::{DeclEngineGet, DeclId}, engine_threading::DebugWithEngines, language::{ parsed::*, ty::{self, TyAstNodeContent, TyDecl}, CallPath, ModName, - }, - semantic_analysis::*, - Engines, TypeInfo, + }, query_engine::TyModuleCacheEntry, semantic_analysis::*, Engines, TypeInfo }; use super::{ @@ -264,6 +261,14 @@ impl ty::TyModule { kind: TreeType, parsed: &ParseModule, ) -> Result { + // Check if the module is already in the cache + if let Some(source_id) = parsed.span.source_id() { + let path = engines.se().get_path(&source_id); + let entry = engines.qe().get_ty_module_cache_entry(&path).unwrap(); + // Return the cached module + return Ok(entry.module); + } + let ParseModule { submodules, tree, @@ -374,13 +379,24 @@ impl ty::TyModule { } } - Ok(Self { + let ty_module = Self { span: span.clone(), submodules, namespace: ctx.namespace.clone(), all_nodes, attributes: attributes.clone(), - }) + }; + + // Cache the ty module + if let Some(source_id) = span.source_id() { + let path = engines.se().get_path(&source_id); + engines.qe().insert_ty_module_cache_entry(TyModuleCacheEntry { + path: Arc::new(path), + module: ty_module.clone(), + }); + } + + Ok(ty_module) } // Filter and gather impl items From c1812d125c4296646d4277a589b878658fd91ed2 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Thu, 11 Jul 2024 13:38:53 +1000 Subject: [PATCH 02/21] add fusd test --- sway-core/src/semantic_analysis/module.rs | 11 +++++++---- sway-lsp/tests/lib.rs | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index d4cc7585d5e..6253638bbe9 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -17,7 +17,7 @@ use crate::{ parsed::*, ty::{self, TyAstNodeContent, TyDecl}, CallPath, ModName, - }, query_engine::TyModuleCacheEntry, semantic_analysis::*, Engines, TypeInfo + }, query_engine::TyModuleCacheEntry, semantic_analysis::*, BuildConfig, Engines, TypeInfo }; use super::{ @@ -264,9 +264,12 @@ impl ty::TyModule { // Check if the module is already in the cache if let Some(source_id) = parsed.span.source_id() { let path = engines.se().get_path(&source_id); - let entry = engines.qe().get_ty_module_cache_entry(&path).unwrap(); - // Return the cached module - return Ok(entry.module); + eprintln!("Checking cache for module {}", path.display()); + if let Some(entry) = engines.qe().get_ty_module_cache_entry(&path) { + eprintln!("Cache hit for module {}", path.display()); + // Return the cached module + return Ok(entry.module); + } } let ParseModule { diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index 6d726675236..b3866d2f175 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -162,6 +162,22 @@ fn did_change() { }); } +#[test] +fn did_open_fluid_libraries() { + run_async!({ + let now = std::time::Instant::now(); + let (mut service, _) = LspService::build(ServerState::new) + .custom_method("sway/metrics", ServerState::metrics) + .finish(); + let uri = init_and_open(&mut service, PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries").join("src/interface.sw")).await; + + let _ = lsp::did_change_request(&mut service, &uri, 1, None).await; + service.inner().wait_for_parsing().await; + eprintln!("Elapsed time: {:?}", now.elapsed()); + shutdown_and_exit(&mut service).await; + }); +} + #[test] fn did_cache_test() { run_async!({ From 05bec68a25315759a6749ffd77f1da7c7eb6805e Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Mon, 15 Jul 2024 13:33:39 +1000 Subject: [PATCH 03/21] check if ty modele cache is up to date --- sway-core/src/lib.rs | 74 ++++++++++- sway-core/src/query_engine/mod.rs | 11 +- sway-core/src/semantic_analysis/module.rs | 148 ++++++++++++++++----- sway-core/src/semantic_analysis/program.rs | 11 +- sway-lsp/tests/lib.rs | 8 +- 5 files changed, 207 insertions(+), 45 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index bdc7e683ceb..4a813b225aa 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -33,7 +33,7 @@ use control_flow_analysis::ControlFlowGraph; pub use debug_generation::write_dwarf; use indexmap::IndexMap; use metadata::MetadataManager; -use query_engine::{ModuleCacheKey, ModulePath, ProgramsCacheEntry}; +use query_engine::{ModuleCacheKey, ProgramsCacheEntry, TyModuleCacheEntry}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; @@ -236,7 +236,7 @@ fn parse_in_memory( pub struct Submodule { name: Ident, - path: ModulePath, + path: Arc, lexed: lexed::LexedSubmodule, parsed: parsed::ParseSubmodule, } @@ -412,13 +412,24 @@ fn parse_module_tree( .and_then(|lsp| lsp.file_versions.get(path.as_ref()).copied()) .unwrap_or(None); let cache_entry = ModuleCacheEntry { - path, + path: path.clone(), modified_time, hash, dependencies, include_tests, version, }; + + let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let relevant_path = path + .iter() + .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) + .collect::(); + eprintln!( + "🀄 🗂️ Inserted cache entry for parse module {:?}", + relevant_path + ); + query_engine.insert_parse_module_cache_entry(cache_entry); Ok(ParsedModuleTree { @@ -428,16 +439,58 @@ fn parse_module_tree( }) } -fn is_parse_module_cache_up_to_date( +pub(crate) fn is_ty_module_cache_up_to_date( + path: &Arc, + build_config: Option<&BuildConfig>, + entry: &TyModuleCacheEntry, +) -> bool { + let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let relevant_path = path + .iter() + .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) + .collect::(); + + let cache_up_to_date = build_config + .as_ref() + .and_then(|x| x.lsp_mode.as_ref()) + .and_then(|lsp| lsp.file_versions.get(path.as_ref())) + .map_or_else( + || false, + |version| !version.map_or(false, |v| v > entry.version.unwrap_or(0)), + ); + + let res = if cache_up_to_date { + entry + .dependencies + .iter() + .all(|path| is_ty_module_cache_up_to_date(path, build_config, entry)) + } else { + false + }; + + eprintln!( + "📟 👓 Checking cache for TY module {:?} | is up to date? {}", + relevant_path, res + ); + res +} + +pub(crate) fn is_parse_module_cache_up_to_date( engines: &Engines, path: &Arc, include_tests: bool, build_config: Option<&BuildConfig>, ) -> bool { + let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let relevant_path = path + .iter() + .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) + .collect::(); + let query_engine = engines.qe(); let key = ModuleCacheKey::new(path.clone(), include_tests); let entry = query_engine.get_parse_module_cache_entry(&key); - match entry { + let res = match entry { Some(entry) => { // Let's check if we can re-use the dependency information // we got from the cache. @@ -480,7 +533,14 @@ fn is_parse_module_cache_up_to_date( } } None => false, - } + }; + + eprintln!( + "🀄 👓 Checking cache for parse module {:?} | is up to date? {}", + relevant_path, res + ); + + res } fn module_path( @@ -696,6 +756,7 @@ pub fn compile_to_ast( package_name: &str, retrigger_compilation: Option>, ) -> Result { + eprintln!("👨‍💻 compile_to_ast 👨‍💻"); check_should_abort(handler, retrigger_compilation.clone())?; let query_engine = engines.qe(); let mut metrics = PerformanceData::default(); @@ -739,6 +800,7 @@ pub fn compile_to_ast( parsed_program.exclude_tests(engines); } + eprintln!("👩‍💻 parsed to typed AST 👩‍💻"); // Type check (+ other static analysis) the CST to a typed AST. let typed_res = time_expr!( "parse the concrete syntax tree (CST) to a typed AST", diff --git a/sway-core/src/query_engine/mod.rs b/sway-core/src/query_engine/mod.rs index bace080fe7f..70326a75555 100644 --- a/sway-core/src/query_engine/mod.rs +++ b/sway-core/src/query_engine/mod.rs @@ -8,8 +8,6 @@ use crate::decl_engine::{DeclId, DeclRef}; use crate::language::ty::{TyFunctionDecl, TyFunctionSig, TyModule}; use crate::{Engines, Programs}; -pub type ModulePath = Arc; - #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ModuleCacheKey { pub path: Arc, @@ -27,12 +25,12 @@ impl ModuleCacheKey { #[derive(Clone, Debug)] pub struct ModuleCacheEntry { - pub path: ModulePath, - pub modified_time: Option, + pub path: Arc, pub hash: u64, - pub dependencies: Vec, pub include_tests: bool, + pub dependencies: Vec>, pub version: Option, + pub modified_time: Option, } pub type ModuleCacheMap = HashMap; @@ -50,7 +48,10 @@ pub type ProgramsCacheMap = HashMap, ProgramsCacheEntry>; pub struct TyModuleCacheEntry { pub path: Arc, pub module: TyModule, + pub dependencies: Vec>, + pub version: Option, } + pub type TyModuleCacheMap = HashMap, TyModuleCacheEntry>; #[derive(Clone, Debug)] diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 6253638bbe9..da8fc34c202 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -1,23 +1,30 @@ use std::{ collections::{HashMap, HashSet}, fmt::Display, - fs, sync::Arc, + fs, + path::PathBuf, + sync::Arc, }; -use fuel_vm::fuel_crypto::coins_bip32::path; use graph_cycles::Cycles; use sway_error::{ error::CompileError, handler::{ErrorEmitted, Handler}, }; -use sway_types::{BaseIdent, Named}; +use sway_types::{BaseIdent, Named, SourceId, Spanned}; use crate::{ - decl_engine::{DeclEngineGet, DeclId}, engine_threading::DebugWithEngines, language::{ + decl_engine::{DeclEngineGet, DeclId}, + engine_threading::DebugWithEngines, + is_parse_module_cache_up_to_date, is_ty_module_cache_up_to_date, + language::{ parsed::*, ty::{self, TyAstNodeContent, TyDecl}, CallPath, ModName, - }, query_engine::TyModuleCacheEntry, semantic_analysis::*, BuildConfig, Engines, TypeInfo + }, + query_engine::TyModuleCacheEntry, + semantic_analysis::*, + BuildConfig, Engines, TypeInfo, }; use super::{ @@ -251,6 +258,43 @@ impl ty::TyModule { Ok(()) } + fn check_cache( + source_id: Option<&SourceId>, + engines: &Engines, + build_config: Option<&BuildConfig>, + ) -> Option { + // Check if the module is already in the cache + if let Some(source_id) = source_id { + let path = engines.se().get_path(&source_id); + + let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let relevant_path = path + .iter() + .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) + .collect::(); + + eprintln!("🥸 Checking cache for TY module {:?}", relevant_path); + if let Some(entry) = engines.qe().get_ty_module_cache_entry(&path) { + // We now need to check if the module is up to date, if not, we need to recompute it so + // we will return None, otherwise we will return the cached module. + + // For now we will duplicate the logic of the parsed cache entry check but + // we should ideally have a way of sharing this logic. + + // Let's check if we can re-use the dependency information + // we got from the cache. + if is_ty_module_cache_up_to_date(&path.into(), build_config, &entry) { + eprintln!("✅ Cache hit for module {:?}", relevant_path); + // Return the cached module + return Some(entry.module); + } else { + eprintln!("🔄 Cache unable to be used for module {:?}", relevant_path); + } + } + } + None + } + /// Type-check the given parsed module to produce a typed module. /// /// Recursively type-checks submodules first. @@ -260,18 +304,8 @@ impl ty::TyModule { engines: &Engines, kind: TreeType, parsed: &ParseModule, + build_config: Option<&BuildConfig>, ) -> Result { - // Check if the module is already in the cache - if let Some(source_id) = parsed.span.source_id() { - let path = engines.se().get_path(&source_id); - eprintln!("Checking cache for module {}", path.display()); - if let Some(entry) = engines.qe().get_ty_module_cache_entry(&path) { - eprintln!("Cache hit for module {}", path.display()); - // Return the cached module - return Ok(entry.module); - } - } - let ParseModule { submodules, tree, @@ -289,20 +323,42 @@ impl ty::TyModule { .iter() .find(|(submod_name, _submodule)| eval_mod_name == submod_name) .unwrap(); - Ok(( - name.clone(), - ty::TySubmodule::type_check( - handler, - ctx.by_ref(), - engines, + + // Check if the submodule cache is up to date + if let Some(module) = ty::TyModule::check_cache( + submodule.module.span.source_id(), + engines, + build_config, + ) { + let submodule = ty::TySubmodule { + module, + mod_name_span: submodule.mod_name_span.clone(), + }; + Ok((name.clone(), submodule)) + } else { + Ok(( name.clone(), - kind, - submodule, - )?, - )) + ty::TySubmodule::type_check( + handler, + ctx.by_ref(), + engines, + name.clone(), + kind, + submodule, + build_config, + )?, + )) + } }) .collect::, _>>(); + // Check if the root module cache is up to date + if let Some(module) = + ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) + { + return Ok(module); + } + // TODO: Ordering should be solved across all modules prior to the beginning of type-check. let ordered_nodes = node_dependencies::order_ast_nodes_by_dependency( handler, @@ -382,6 +438,16 @@ impl ty::TyModule { } } + let dependencies = submodules + .iter() + .filter_map(|(ident, _)| { + ident + .span() + .source_id() + .map(|path| Arc::new(engines.se().get_path(path))) + }) + .collect::>(); + let ty_module = Self { span: span.clone(), submodules, @@ -390,13 +456,29 @@ impl ty::TyModule { attributes: attributes.clone(), }; - // Cache the ty module + // Cache the ty module if let Some(source_id) = span.source_id() { let path = engines.se().get_path(&source_id); - engines.qe().insert_ty_module_cache_entry(TyModuleCacheEntry { - path: Arc::new(path), - module: ty_module.clone(), - }); + let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let relevant_path = path + .iter() + .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) + .collect::(); + eprintln!("💾 Inserting cache entry for TY module {:?}", relevant_path); + + let version = build_config + .and_then(|config| config.lsp_mode.as_ref()) + .and_then(|lsp| lsp.file_versions.get(&path).copied()) + .flatten(); + + engines + .qe() + .insert_ty_module_cache_entry(TyModuleCacheEntry { + path: Arc::new(path), + module: ty_module.clone(), + dependencies, + version, + }); } Ok(ty_module) @@ -614,6 +696,7 @@ impl ty::TySubmodule { mod_name: ModName, kind: TreeType, submodule: &ParseSubmodule, + build_config: Option<&BuildConfig>, ) -> Result { let ParseSubmodule { module, @@ -621,7 +704,8 @@ impl ty::TySubmodule { visibility, } = submodule; parent_ctx.enter_submodule(mod_name, *visibility, module.span.clone(), |submod_ctx| { - let module_res = ty::TyModule::type_check(handler, submod_ctx, engines, kind, module); + let module_res = + ty::TyModule::type_check(handler, submod_ctx, engines, kind, module, build_config); module_res.map(|module| ty::TySubmodule { module, mod_name_span: mod_name_span.clone(), diff --git a/sway-core/src/semantic_analysis/program.rs b/sway-core/src/semantic_analysis/program.rs index 71f73f359d7..65f5d762552 100644 --- a/sway-core/src/semantic_analysis/program.rs +++ b/sway-core/src/semantic_analysis/program.rs @@ -60,7 +60,16 @@ impl TyProgram { let ParseProgram { root, kind } = parsed; - let root = ty::TyModule::type_check(handler, ctx.by_ref(), engines, parsed.kind, root)?; + let now = std::time::Instant::now(); + let root = ty::TyModule::type_check( + handler, + ctx.by_ref(), + engines, + parsed.kind, + root, + build_config, + )?; + eprintln!("⏱️ Type-checking module took {:?}", now.elapsed()); let (kind, declarations, configurables) = Self::validate_root( handler, diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index b3866d2f175..9f3faeb974d 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -169,8 +169,14 @@ fn did_open_fluid_libraries() { let (mut service, _) = LspService::build(ServerState::new) .custom_method("sway/metrics", ServerState::metrics) .finish(); - let uri = init_and_open(&mut service, PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries").join("src/interface.sw")).await; + let uri = init_and_open( + &mut service, + PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries") + .join("src/interface.sw"), + ) + .await; + eprintln!("\n 🪆🪆🪆🪆 Initial compilation complete, starting recompilation 🪆🪆🪆🪆 \n"); let _ = lsp::did_change_request(&mut service, &uri, 1, None).await; service.inner().wait_for_parsing().await; eprintln!("Elapsed time: {:?}", now.elapsed()); From b1f8432e9f930604904fc38c7514af02f2219641 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Mon, 15 Jul 2024 13:58:15 +1000 Subject: [PATCH 04/21] add clear module fns for GC --- sway-core/src/decl_engine/engine.rs | 49 +++++++++++++++++++++- sway-core/src/decl_engine/parsed_engine.rs | 43 ++++++++++++++++++- sway-core/src/engine_threading.rs | 8 ++++ sway-core/src/type_system/engine.rs | 22 ++++++---- sway-lsp/src/core/session.rs | 8 ++++ sway-lsp/src/server_state.rs | 3 +- 6 files changed, 122 insertions(+), 11 deletions(-) diff --git a/sway-core/src/decl_engine/engine.rs b/sway-core/src/decl_engine/engine.rs index 8f992f119bd..e1f8843385e 100644 --- a/sway-core/src/decl_engine/engine.rs +++ b/sway-core/src/decl_engine/engine.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, }; -use sway_types::{Named, ProgramId, Spanned}; +use sway_types::{Named, ProgramId, SourceId, Spanned}; use crate::{ concurrent_slab::ConcurrentSlab, @@ -375,6 +375,53 @@ decl_engine_clear_program!( type_alias_slab, ty::TyTypeAliasDecl; ); +macro_rules! decl_engine_clear_module { + ($($slab:ident, $decl:ty);* $(;)?) => { + impl DeclEngine { + pub fn clear_module(&mut self, source_id: &SourceId) { + self.parents.write().retain(|key, _| { + match key { + AssociatedItemDeclId::TraitFn(decl_id) => { + self.get_trait_fn(decl_id).span().source_id().map_or(true, |src_id| src_id != source_id) + }, + AssociatedItemDeclId::Function(decl_id) => { + self.get_function(decl_id).span().source_id().map_or(true, |src_id| src_id != source_id) + }, + AssociatedItemDeclId::Type(decl_id) => { + self.get_type(decl_id).span().source_id().map_or(true, |src_id| src_id != source_id) + }, + AssociatedItemDeclId::Constant(decl_id) => { + self.get_constant(decl_id).span().source_id().map_or(true, |src_id| src_id != source_id) + }, + } + }); + + $( + self.$slab.retain(|_k, ty| match ty.span().source_id() { + Some(src_id) => src_id != source_id, + None => true, + }); + )* + } + } + }; +} + +decl_engine_clear_module!( + function_slab, ty::TyFunctionDecl; + trait_slab, ty::TyTraitDecl; + trait_fn_slab, ty::TyTraitFn; + trait_type_slab, ty::TyTraitType; + impl_self_or_trait_slab, ty::TyImplTrait; + struct_slab, ty::TyStructDecl; + storage_slab, ty::TyStorageDecl; + abi_slab, ty::TyAbiDecl; + constant_slab, ty::TyConstantDecl; + configurable_slab, ty::TyConfigurableDecl; + enum_slab, ty::TyEnumDecl; + type_alias_slab, ty::TyTypeAliasDecl; +); + impl DeclEngine { /// Given a [DeclRef] `index`, finds all the parents of `index` and all the /// recursive parents of those parents, and so on. Does not perform diff --git a/sway-core/src/decl_engine/parsed_engine.rs b/sway-core/src/decl_engine/parsed_engine.rs index 8b8f84107eb..a3e2ffdf230 100644 --- a/sway-core/src/decl_engine/parsed_engine.rs +++ b/sway-core/src/decl_engine/parsed_engine.rs @@ -9,7 +9,7 @@ use crate::{ }; use std::sync::Arc; -use sway_types::{ProgramId, Spanned}; +use sway_types::{ProgramId, SourceId, Spanned}; use super::parsed_id::ParsedDeclId; @@ -167,6 +167,47 @@ decl_engine_clear_program!( .span()), ); + +macro_rules! decl_engine_clear_module { + ($(($slab:ident, $getter:expr)),* $(,)?) => { + impl ParsedDeclEngine { + pub fn clear_module(&mut self, program_id: &SourceId) { + $( + self.$slab.retain(|_k, item| { + #[allow(clippy::redundant_closure_call)] + let span = $getter(item); + match span.source_id() { + Some(src_id) => src_id != program_id, + None => true, + } + }); + )* + } + } + }; +} + +decl_engine_clear_module!( + (variable_slab, |item: &VariableDeclaration| item.name.span()), + (function_slab, |item: &FunctionDeclaration| item.name.span()), + (trait_slab, |item: &TraitDeclaration| item.name.span()), + (trait_fn_slab, |item: &TraitFn| item.name.span()), + (trait_type_slab, |item: &TraitTypeDeclaration| item + .name + .span()), + (impl_self_or_trait_slab, |item: &ImplSelfOrTrait| item + .block_span + .clone()), + (struct_slab, |item: &StructDeclaration| item.name.span()), + (storage_slab, |item: &StorageDeclaration| item.span.clone()), + (abi_slab, |item: &AbiDeclaration| item.name.span()), + (constant_slab, |item: &ConstantDeclaration| item.name.span()), + (enum_slab, |item: &EnumDeclaration| item.name.span()), + (type_alias_slab, |item: &TypeAliasDeclaration| item + .name + .span()), +); + impl ParsedDeclEngine { /// Friendly helper method for calling the `get` method from the /// implementation of [ParsedDeclEngineGet] for [ParsedDeclEngine] diff --git a/sway-core/src/engine_threading.rs b/sway-core/src/engine_threading.rs index c698d9700f4..e0e151249eb 100644 --- a/sway-core/src/engine_threading.rs +++ b/sway-core/src/engine_threading.rs @@ -48,6 +48,14 @@ impl Engines { self.parsed_decl_engine.clear_program(program_id); } + /// Removes all data associated with `source_id` from the declaration and type engines. + /// It is intended to be used during garbage collection to remove any data that is no longer needed. + pub fn clear_module(&mut self, source_id: &sway_types::SourceId) { + self.type_engine.clear_module(source_id); + self.decl_engine.clear_module(source_id); + self.parsed_decl_engine.clear_module(source_id); + } + /// Helps out some `thing: T` by adding `self` as context. pub fn help_out(&self, thing: T) -> WithEngines<'_, T> { WithEngines { diff --git a/sway-core/src/type_system/engine.rs b/sway-core/src/type_system/engine.rs index 38871fd4108..9e2085bda39 100644 --- a/sway-core/src/type_system/engine.rs +++ b/sway-core/src/type_system/engine.rs @@ -67,16 +67,22 @@ impl TypeEngine { } } + fn clear_items(&mut self, keep: F) + where + F: Fn(&SourceId) -> bool, + { + self.slab.retain(|_, tsi| tsi.source_id.as_ref().map_or(true, &keep)); + self.id_map.write().retain(|tsi, _| tsi.source_id.as_ref().map_or(true, &keep)); + } + /// Removes all data associated with `program_id` from the type engine. pub fn clear_program(&mut self, program_id: &ProgramId) { - self.slab.retain(|_, tsi| match tsi.source_id { - Some(source_id) => &source_id.program_id() != program_id, - None => true, - }); - self.id_map.write().retain(|tsi, _| match tsi.source_id { - Some(source_id) => &source_id.program_id() != program_id, - None => true, - }); + self.clear_items(|id| id.program_id() != *program_id); + } + + /// Removes all data associated with `source_id` from the type engine. + pub fn clear_module(&mut self, source_id: &SourceId) { + self.clear_items(|id| id != source_id); } pub fn replace(&self, id: TypeId, new_value: TypeSourceInfo) { diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index 6dab2952254..0b622b75899 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -134,6 +134,14 @@ impl Session { Ok(()) } + pub fn garbage_collect_module(&self, engines: &mut Engines, uri: &Url) -> Result<(), LanguageServerError> { + let path = uri.to_file_path().unwrap(); + eprintln!("🗑️ Garbage collecting module {:?}", path); + let source_id = { engines.se().get_source_id(&path) }; + engines.clear_module(&source_id); + Ok(()) + } + pub fn token_ranges(&self, url: &Url, position: Position) -> Option> { let _p = tracing::trace_span!("token_ranges").entered(); let mut token_ranges: Vec<_> = self diff --git a/sway-lsp/src/server_state.rs b/sway-lsp/src/server_state.rs index 2f614e93072..d5dd3b69040 100644 --- a/sway-lsp/src/server_state.rs +++ b/sway-lsp/src/server_state.rs @@ -140,7 +140,8 @@ impl ServerState { { // Call this on the engines clone so we don't clear types that are still in use // and might be needed in the case cancel compilation was triggered. - if let Err(err) = session.garbage_collect(&mut engines_clone) { + // if let Err(err) = session.garbage_collect_program(&mut engines_clone) { + if let Err(err) = session.garbage_collect_module(&mut engines_clone, &uri) { tracing::error!( "Unable to perform garbage collection: {}", err.to_string() From cea3ca235eb7d845e8da9562dd3a980580a10410 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Mon, 15 Jul 2024 17:07:33 +1000 Subject: [PATCH 05/21] fix stack overflow --- sway-core/src/lib.rs | 70 +++++++++++++---------- sway-core/src/semantic_analysis/module.rs | 57 ++++++++++++++---- sway-lsp/src/core/session.rs | 11 +++- sway-lsp/src/server_state.rs | 4 +- 4 files changed, 97 insertions(+), 45 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 4a813b225aa..1b3afd8fbf2 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -408,6 +408,7 @@ fn parse_module_tree( .ok() .and_then(|m| m.modified().ok()); let dependencies = submodules.into_iter().map(|s| s.path).collect::>(); + eprintln!("🐔 path {:?} | dependencies {:?}", path, dependencies); let version = lsp_mode .and_then(|lsp| lsp.file_versions.get(path.as_ref()).copied()) .unwrap_or(None); @@ -440,39 +441,45 @@ fn parse_module_tree( } pub(crate) fn is_ty_module_cache_up_to_date( + engines: &Engines, path: &Arc, build_config: Option<&BuildConfig>, - entry: &TyModuleCacheEntry, + //entry: &TyModuleCacheEntry, ) -> bool { - let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; - let relevant_path = path - .iter() - .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) - .collect::(); - - let cache_up_to_date = build_config - .as_ref() - .and_then(|x| x.lsp_mode.as_ref()) - .and_then(|lsp| lsp.file_versions.get(path.as_ref())) - .map_or_else( - || false, - |version| !version.map_or(false, |v| v > entry.version.unwrap_or(0)), - ); - - let res = if cache_up_to_date { - entry - .dependencies - .iter() - .all(|path| is_ty_module_cache_up_to_date(path, build_config, entry)) - } else { - false - }; - - eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? {}", - relevant_path, res - ); - res + let query_engine = engines.qe(); + let entry = query_engine.get_ty_module_cache_entry(&path); + match entry { + Some(entry) => { + // return false; + let cache_up_to_date = build_config + .as_ref() + .and_then(|x| x.lsp_mode.as_ref()) + .and_then(|lsp| lsp.file_versions.get(path.as_ref())) + .map_or_else( + || false, + |version| !version.map_or(false, |v| v > entry.version.unwrap_or(0)), + ); + // Only putting this here to confirm we don't get a stack overflow if we return early. + // return cache_up_to_date; + if cache_up_to_date { + // This is causing a stack overflow, why? + eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); + entry + .dependencies + .iter() + .all(|path| { + eprint!("checking dep path {:?} ", path); + is_ty_module_cache_up_to_date(engines, path, build_config) + }) + } else { + false + } + } + None => { + false + } + } + } pub(crate) fn is_parse_module_cache_up_to_date( @@ -525,7 +532,10 @@ pub(crate) fn is_parse_module_cache_up_to_date( // Look at the dependencies recursively to make sure they have not been // modified either. if cache_up_to_date { + //eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); + entry.dependencies.iter().all(|path| { +// eprint!("checking dep path {:?} ", path); is_parse_module_cache_up_to_date(engines, path, include_tests, build_config) }) } else { diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index da8fc34c202..039a967782c 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -22,7 +22,7 @@ use crate::{ ty::{self, TyAstNodeContent, TyDecl}, CallPath, ModName, }, - query_engine::TyModuleCacheEntry, + query_engine::{ModuleCacheKey, TyModuleCacheEntry}, semantic_analysis::*, BuildConfig, Engines, TypeInfo, }; @@ -272,8 +272,8 @@ impl ty::TyModule { .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - eprintln!("🥸 Checking cache for TY module {:?}", relevant_path); + if let Some(entry) = engines.qe().get_ty_module_cache_entry(&path) { // We now need to check if the module is up to date, if not, we need to recompute it so // we will return None, otherwise we will return the cached module. @@ -283,11 +283,22 @@ impl ty::TyModule { // Let's check if we can re-use the dependency information // we got from the cache. - if is_ty_module_cache_up_to_date(&path.into(), build_config, &entry) { + + if is_ty_module_cache_up_to_date(engines, &path.into(), build_config) { + eprintln!( + "📟 👓 Checking cache for TY module {:?} | is up to date? {}", + relevant_path, true + ); + eprintln!("✅ Cache hit for module {:?}", relevant_path); // Return the cached module return Some(entry.module); } else { + eprintln!( + "📟 👓 Checking cache for TY module {:?} | is up to date? {}", + relevant_path, false + ); + eprintln!("🔄 Cache unable to be used for module {:?}", relevant_path); } } @@ -315,6 +326,7 @@ impl ty::TyModule { .. } = parsed; + // Type-check submodules first in order of evaluation previously computed by the dependency graph. let submodules_res = module_eval_order .iter() @@ -438,15 +450,15 @@ impl ty::TyModule { } } - let dependencies = submodules - .iter() - .filter_map(|(ident, _)| { - ident - .span() - .source_id() - .map(|path| Arc::new(engines.se().get_path(path))) - }) - .collect::>(); + // let dependencies = submodules + // .iter() + // .filter_map(|(ident, _)| { + // ident + // .span() + // .source_id() + // .map(|src_id| Arc::new(engines.se().get_path(src_id))) + // }) + // .collect::>(); let ty_module = Self { span: span.clone(), @@ -458,12 +470,33 @@ impl ty::TyModule { // Cache the ty module if let Some(source_id) = span.source_id() { + let path = engines.se().get_path(&source_id); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); + + // let module_eval_order = module_eval_order.iter().map(|ident| { + // let path = engines.se().get_path(ident.span().source_id().unwrap()); + // }).collect::>(); + + // let module_eval_order = module_eval_order.iter() + // .filter_map(|ident| { + // ident.span().source_id() + // .map(|source_id| engines.se().get_path(source_id)) + // }) + // .collect::>(); + + let dependencies = engines.qe().get_parse_module_cache_entry(&ModuleCacheKey { + path: path.clone().into(), + include_tests: true, // TODO: pass this in + }).unwrap().dependencies; + + //eprintln!("🐤 🐤 🐤 🐤 🐤 🐤 🐤 🐤 path {:?} | module_eval_order {:?}", relevant_path, module_eval_order); + eprintln!("🐤 path {:?} | dependencies {:?}", relevant_path, dependencies); + eprintln!("💾 Inserting cache entry for TY module {:?}", relevant_path); let version = build_config diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index 0b622b75899..0236e278051 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -124,7 +124,10 @@ impl Session { } /// Clean up memory in the [TypeEngine] and [DeclEngine] for the user's workspace. - pub fn garbage_collect(&self, engines: &mut Engines) -> Result<(), LanguageServerError> { + pub fn garbage_collect_program( + &self, + engines: &mut Engines, + ) -> Result<(), LanguageServerError> { let _p = tracing::trace_span!("garbage_collect").entered(); let path = self.sync.temp_dir()?; let program_id = { engines.se().get_program_id(&path) }; @@ -134,7 +137,11 @@ impl Session { Ok(()) } - pub fn garbage_collect_module(&self, engines: &mut Engines, uri: &Url) -> Result<(), LanguageServerError> { + pub fn garbage_collect_module( + &self, + engines: &mut Engines, + uri: &Url, + ) -> Result<(), LanguageServerError> { let path = uri.to_file_path().unwrap(); eprintln!("🗑️ Garbage collecting module {:?}", path); let source_id = { engines.se().get_source_id(&path) }; diff --git a/sway-lsp/src/server_state.rs b/sway-lsp/src/server_state.rs index d5dd3b69040..57ab7cbf0d3 100644 --- a/sway-lsp/src/server_state.rs +++ b/sway-lsp/src/server_state.rs @@ -141,7 +141,9 @@ impl ServerState { // Call this on the engines clone so we don't clear types that are still in use // and might be needed in the case cancel compilation was triggered. // if let Err(err) = session.garbage_collect_program(&mut engines_clone) { - if let Err(err) = session.garbage_collect_module(&mut engines_clone, &uri) { + if let Err(err) = + session.garbage_collect_module(&mut engines_clone, &uri) + { tracing::error!( "Unable to perform garbage collection: {}", err.to_string() From c21a29d6ea09f735def913841e04104c1e8d5d0f Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Mon, 15 Jul 2024 17:13:57 +1000 Subject: [PATCH 06/21] clean up --- sway-core/src/lib.rs | 6 ------ sway-core/src/semantic_analysis/module.rs | 23 +---------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 1b3afd8fbf2..2f633a46da2 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -444,13 +444,11 @@ pub(crate) fn is_ty_module_cache_up_to_date( engines: &Engines, path: &Arc, build_config: Option<&BuildConfig>, - //entry: &TyModuleCacheEntry, ) -> bool { let query_engine = engines.qe(); let entry = query_engine.get_ty_module_cache_entry(&path); match entry { Some(entry) => { - // return false; let cache_up_to_date = build_config .as_ref() .and_then(|x| x.lsp_mode.as_ref()) @@ -459,11 +457,7 @@ pub(crate) fn is_ty_module_cache_up_to_date( || false, |version| !version.map_or(false, |v| v > entry.version.unwrap_or(0)), ); - // Only putting this here to confirm we don't get a stack overflow if we return early. - // return cache_up_to_date; if cache_up_to_date { - // This is causing a stack overflow, why? - eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); entry .dependencies .iter() diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 039a967782c..802f26d1d2f 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -450,16 +450,6 @@ impl ty::TyModule { } } - // let dependencies = submodules - // .iter() - // .filter_map(|(ident, _)| { - // ident - // .span() - // .source_id() - // .map(|src_id| Arc::new(engines.se().get_path(src_id))) - // }) - // .collect::>(); - let ty_module = Self { span: span.clone(), submodules, @@ -470,7 +460,6 @@ impl ty::TyModule { // Cache the ty module if let Some(source_id) = span.source_id() { - let path = engines.se().get_path(&source_id); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; let relevant_path = path @@ -478,17 +467,7 @@ impl ty::TyModule { .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - // let module_eval_order = module_eval_order.iter().map(|ident| { - // let path = engines.se().get_path(ident.span().source_id().unwrap()); - // }).collect::>(); - - // let module_eval_order = module_eval_order.iter() - // .filter_map(|ident| { - // ident.span().source_id() - // .map(|source_id| engines.se().get_path(source_id)) - // }) - // .collect::>(); - + // how about instead of saving this here we just check in the is_ty_module_cache_up_to_date function let dependencies = engines.qe().get_parse_module_cache_entry(&ModuleCacheKey { path: path.clone().into(), include_tests: true, // TODO: pass this in From 6708cc762d10b9ca6a113b9180b4db87255cce15 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Tue, 16 Jul 2024 15:07:23 +1000 Subject: [PATCH 07/21] rework so we only need one ModuleCacheEntry for both parsed and typed --- sway-core/src/decl_engine/parsed_engine.rs | 1 - sway-core/src/lib.rs | 85 ++++++----- sway-core/src/query_engine/mod.rs | 155 +++++++++++++++------ sway-core/src/semantic_analysis/module.rs | 78 ++++++----- sway-core/src/type_system/engine.rs | 7 +- 5 files changed, 207 insertions(+), 119 deletions(-) diff --git a/sway-core/src/decl_engine/parsed_engine.rs b/sway-core/src/decl_engine/parsed_engine.rs index a3e2ffdf230..642f0c9ee6e 100644 --- a/sway-core/src/decl_engine/parsed_engine.rs +++ b/sway-core/src/decl_engine/parsed_engine.rs @@ -167,7 +167,6 @@ decl_engine_clear_program!( .span()), ); - macro_rules! decl_engine_clear_module { ($(($slab:ident, $getter:expr)),* $(,)?) => { impl ParsedDeclEngine { diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 2f633a46da2..24df5a46d75 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -31,9 +31,10 @@ pub use asm_generation::{CompiledBytecode, FinalizedEntry}; pub use build_config::{BuildConfig, BuildTarget, LspConfig, OptLevel, PrintAsm, PrintIr}; use control_flow_analysis::ControlFlowGraph; pub use debug_generation::write_dwarf; +use fuel_vm::fuel_merkle::common; use indexmap::IndexMap; use metadata::MetadataManager; -use query_engine::{ModuleCacheKey, ProgramsCacheEntry, TyModuleCacheEntry}; +use query_engine::{ModuleCacheKey, ModuleCommonInfo, ParsedModuleInfo, ProgramsCacheEntry}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; @@ -412,14 +413,26 @@ fn parse_module_tree( let version = lsp_mode .and_then(|lsp| lsp.file_versions.get(path.as_ref()).copied()) .unwrap_or(None); - let cache_entry = ModuleCacheEntry { + // let cache_entry = ModuleCacheEntry { + // path: path.clone(), + // modified_time, + // hash, + // dependencies, + // include_tests, + // version, + // }; + + let common_info = ModuleCommonInfo { path: path.clone(), - modified_time, - hash, - dependencies, include_tests, + dependencies, + hash, + }; + let parsed_info = ParsedModuleInfo { + modified_time, version, }; + let cache_entry = ModuleCacheEntry::new(common_info, parsed_info); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; let relevant_path = path @@ -431,7 +444,7 @@ fn parse_module_tree( relevant_path ); - query_engine.insert_parse_module_cache_entry(cache_entry); + query_engine.update_or_insert_parsed_module_cache_entry(cache_entry); Ok(ParsedModuleTree { tree_type: kind, @@ -440,40 +453,40 @@ fn parse_module_tree( }) } +// TODO: This function can probably be written better and more concisely. pub(crate) fn is_ty_module_cache_up_to_date( engines: &Engines, path: &Arc, + include_tests: bool, build_config: Option<&BuildConfig>, ) -> bool { let query_engine = engines.qe(); - let entry = query_engine.get_ty_module_cache_entry(&path); + let key = ModuleCacheKey::new(path.clone(), include_tests); + let entry = query_engine.get_module_cache_entry(&key); match entry { - Some(entry) => { - let cache_up_to_date = build_config - .as_ref() - .and_then(|x| x.lsp_mode.as_ref()) - .and_then(|lsp| lsp.file_versions.get(path.as_ref())) - .map_or_else( - || false, - |version| !version.map_or(false, |v| v > entry.version.unwrap_or(0)), - ); - if cache_up_to_date { - entry - .dependencies - .iter() - .all(|path| { + Some(entry) => match entry.typed { + Some(typed) => { + let cache_up_to_date = build_config + .as_ref() + .and_then(|x| x.lsp_mode.as_ref()) + .and_then(|lsp| lsp.file_versions.get(path.as_ref())) + .map_or_else( + || false, + |version| !version.map_or(false, |v| v > typed.version.unwrap_or(0)), + ); + if cache_up_to_date { + entry.common.dependencies.iter().all(|path| { eprint!("checking dep path {:?} ", path); - is_ty_module_cache_up_to_date(engines, path, build_config) - }) - } else { - false + is_ty_module_cache_up_to_date(engines, path, include_tests, build_config) + }) + } else { + false + } } - } - None => { - false - } + None => false, + }, + None => false, } - } pub(crate) fn is_parse_module_cache_up_to_date( @@ -490,7 +503,7 @@ pub(crate) fn is_parse_module_cache_up_to_date( let query_engine = engines.qe(); let key = ModuleCacheKey::new(path.clone(), include_tests); - let entry = query_engine.get_parse_module_cache_entry(&key); + let entry = query_engine.get_module_cache_entry(&key); let res = match entry { Some(entry) => { // Let's check if we can re-use the dependency information @@ -509,17 +522,17 @@ pub(crate) fn is_parse_module_cache_up_to_date( let modified_time = std::fs::metadata(path.as_path()) .ok() .and_then(|m| m.modified().ok()); - entry.modified_time == modified_time || { + entry.parsed.modified_time == modified_time || { let src = std::fs::read_to_string(path.as_path()).unwrap(); let mut hasher = DefaultHasher::new(); src.hash(&mut hasher); let hash = hasher.finish(); - hash == entry.hash + hash == entry.common.hash } }, |version| { // The cache is invalid if the lsp version is greater than the last compilation - !version.map_or(false, |v| v > entry.version.unwrap_or(0)) + !version.map_or(false, |v| v > entry.parsed.version.unwrap_or(0)) }, ); @@ -528,8 +541,8 @@ pub(crate) fn is_parse_module_cache_up_to_date( if cache_up_to_date { //eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); - entry.dependencies.iter().all(|path| { -// eprint!("checking dep path {:?} ", path); + entry.common.dependencies.iter().all(|path| { + // eprint!("checking dep path {:?} ", path); is_parse_module_cache_up_to_date(engines, path, include_tests, build_config) }) } else { diff --git a/sway-core/src/query_engine/mod.rs b/sway-core/src/query_engine/mod.rs index 70326a75555..79e5aa04bb0 100644 --- a/sway-core/src/query_engine/mod.rs +++ b/sway-core/src/query_engine/mod.rs @@ -1,12 +1,19 @@ use parking_lot::RwLock; -use std::{collections::HashMap, path::PathBuf, sync::Arc, time::SystemTime}; -use sway_error::error::CompileError; -use sway_error::warning::CompileWarning; +use std::{ + collections::HashMap, + ops::{Deref, DerefMut}, + path::PathBuf, + sync::Arc, + time::SystemTime, +}; +use sway_error::{error::CompileError, warning::CompileWarning}; use sway_types::IdentUnique; -use crate::decl_engine::{DeclId, DeclRef}; -use crate::language::ty::{TyFunctionDecl, TyFunctionSig, TyModule}; -use crate::{Engines, Programs}; +use crate::{ + decl_engine::{DeclId, DeclRef}, + language::ty::{TyFunctionDecl, TyFunctionSig, TyModule}, + {Engines, Programs}, +}; #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ModuleCacheKey { @@ -24,65 +31,139 @@ impl ModuleCacheKey { } #[derive(Clone, Debug)] -pub struct ModuleCacheEntry { +pub struct ModuleCommonInfo { pub path: Arc, pub hash: u64, pub include_tests: bool, pub dependencies: Vec>, - pub version: Option, +} + +#[derive(Clone, Debug)] +pub struct ParsedModuleInfo { pub modified_time: Option, + pub version: Option, } -pub type ModuleCacheMap = HashMap; +#[derive(Clone, Debug)] +pub struct TypedModuleInfo { + pub module: TyModule, + pub modified_time: Option, + pub version: Option, +} #[derive(Clone, Debug)] -pub struct ProgramsCacheEntry { - pub path: Arc, - pub programs: Programs, - pub handler_data: (Vec, Vec), +pub struct ModuleCacheEntry { + pub common: ModuleCommonInfo, + pub parsed: ParsedModuleInfo, + pub typed: Option, +} + +impl ModuleCacheEntry { + pub fn new(common: ModuleCommonInfo, parsed: ParsedModuleInfo) -> Self { + Self { + common, + parsed, + typed: None, + } + } + + pub fn is_typed(&self) -> bool { + self.typed.is_some() + } + + pub fn set_typed(&mut self, typed: TypedModuleInfo) { + self.typed = Some(typed); + } + + pub fn update_common(&mut self, new_common: ModuleCommonInfo) { + self.common = new_common; + } + + pub fn update_parsed(&mut self, new_parsed: ParsedModuleInfo) { + self.parsed = new_parsed; + } + + pub fn update_parsed_and_common( + &mut self, + new_common: ModuleCommonInfo, + new_parsed: ParsedModuleInfo, + ) { + self.common = new_common; + self.parsed = new_parsed; + } +} + +#[derive(Debug, Default, Clone)] +struct ModuleCacheMap(HashMap); + +impl Deref for ModuleCacheMap { + type Target = HashMap; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for ModuleCacheMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl ModuleCacheMap { + pub fn update_entry( + &mut self, + key: &ModuleCacheKey, + new_common: ModuleCommonInfo, + new_parsed: ParsedModuleInfo, + ) { + if let Some(entry) = self.get_mut(key) { + entry.update_parsed_and_common(new_common, new_parsed); + } else { + self.insert(key.clone(), ModuleCacheEntry::new(new_common, new_parsed)); + } + } } pub type ProgramsCacheMap = HashMap, ProgramsCacheEntry>; +pub type FunctionsCacheMap = HashMap<(IdentUnique, String), FunctionCacheEntry>; #[derive(Clone, Debug)] -pub struct TyModuleCacheEntry { +pub struct ProgramsCacheEntry { pub path: Arc, - pub module: TyModule, - pub dependencies: Vec>, - pub version: Option, + pub programs: Programs, + pub handler_data: (Vec, Vec), } -pub type TyModuleCacheMap = HashMap, TyModuleCacheEntry>; - #[derive(Clone, Debug)] pub struct FunctionCacheEntry { pub fn_decl: DeclRef>, } -pub type FunctionsCacheMap = HashMap<(IdentUnique, String), FunctionCacheEntry>; - #[derive(Debug, Default, Clone)] pub struct QueryEngine { // We want the below types wrapped in Arcs to optimize cloning from LSP. - parse_module_cache: Arc>, + module_cache: Arc>, programs_cache: Arc>, function_cache: Arc>, - - ty_module_cache: Arc>, } impl QueryEngine { - pub fn get_parse_module_cache_entry(&self, path: &ModuleCacheKey) -> Option { - let cache = self.parse_module_cache.read(); - cache.get(path).cloned() + pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option { + let cache = self.module_cache.read(); + cache.get(key).cloned() } - pub fn insert_parse_module_cache_entry(&self, entry: ModuleCacheEntry) { - let path = entry.path.clone(); - let include_tests = entry.include_tests; + pub fn update_or_insert_parsed_module_cache_entry(&self, entry: ModuleCacheEntry) { + let path = entry.common.path.clone(); + let include_tests = entry.common.include_tests; let key = ModuleCacheKey::new(path, include_tests); - let mut cache = self.parse_module_cache.write(); - cache.insert(key, entry); + let mut cache = self.module_cache.write(); + cache.update_entry(&key, entry.common, entry.parsed); + } + + pub fn update_typed_module_cache_entry(&self, key: &ModuleCacheKey, entry: TypedModuleInfo) { + let mut cache = self.module_cache.write(); + cache.get_mut(key).unwrap().set_typed(entry); } pub fn get_programs_cache_entry(&self, path: &Arc) -> Option { @@ -95,16 +176,6 @@ impl QueryEngine { cache.insert(entry.path.clone(), entry); } - pub fn get_ty_module_cache_entry(&self, path: &PathBuf) -> Option { - let cache = self.ty_module_cache.read(); - cache.get(path).cloned() - } - - pub fn insert_ty_module_cache_entry(&self, entry: TyModuleCacheEntry) { - let mut cache = self.ty_module_cache.write(); - cache.insert(entry.path.clone(), entry); - } - pub fn get_function( &self, engines: &Engines, diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 802f26d1d2f..26131b08eef 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -22,7 +22,7 @@ use crate::{ ty::{self, TyAstNodeContent, TyDecl}, CallPath, ModName, }, - query_engine::{ModuleCacheKey, TyModuleCacheEntry}, + query_engine::{ModuleCacheKey, TypedModuleInfo}, semantic_analysis::*, BuildConfig, Engines, TypeInfo, }; @@ -266,6 +266,7 @@ impl ty::TyModule { // Check if the module is already in the cache if let Some(source_id) = source_id { let path = engines.se().get_path(&source_id); + let include_tests = build_config.map_or(false, |x| x.include_tests); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; let relevant_path = path @@ -273,8 +274,9 @@ impl ty::TyModule { .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); eprintln!("🥸 Checking cache for TY module {:?}", relevant_path); - - if let Some(entry) = engines.qe().get_ty_module_cache_entry(&path) { + + let key: ModuleCacheKey = ModuleCacheKey::new(path.clone().into(), include_tests); + if let Some(entry) = engines.qe().get_module_cache_entry(&key) { // We now need to check if the module is up to date, if not, we need to recompute it so // we will return None, otherwise we will return the cached module. @@ -283,23 +285,29 @@ impl ty::TyModule { // Let's check if we can re-use the dependency information // we got from the cache. - - if is_ty_module_cache_up_to_date(engines, &path.into(), build_config) { - eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? {}", - relevant_path, true - ); - - eprintln!("✅ Cache hit for module {:?}", relevant_path); - // Return the cached module - return Some(entry.module); - } else { - eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? {}", - relevant_path, false - ); - - eprintln!("🔄 Cache unable to be used for module {:?}", relevant_path); + if let Some(typed) = entry.typed { + if is_ty_module_cache_up_to_date( + engines, + &path.into(), + include_tests, + build_config, + ) { + eprintln!( + "📟 👓 Checking cache for TY module {:?} | is up to date? {}", + relevant_path, true + ); + + eprintln!("✅ Cache hit for module {:?}", relevant_path); + // Return the cached module + return Some(typed.module); + } else { + eprintln!( + "📟 👓 Checking cache for TY module {:?} | is up to date? {}", + relevant_path, false + ); + + eprintln!("🔄 Cache unable to be used for module {:?}", relevant_path); + } } } } @@ -326,7 +334,6 @@ impl ty::TyModule { .. } = parsed; - // Type-check submodules first in order of evaluation previously computed by the dependency graph. let submodules_res = module_eval_order .iter() @@ -467,30 +474,25 @@ impl ty::TyModule { .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - // how about instead of saving this here we just check in the is_ty_module_cache_up_to_date function - let dependencies = engines.qe().get_parse_module_cache_entry(&ModuleCacheKey { - path: path.clone().into(), - include_tests: true, // TODO: pass this in - }).unwrap().dependencies; - - //eprintln!("🐤 🐤 🐤 🐤 🐤 🐤 🐤 🐤 path {:?} | module_eval_order {:?}", relevant_path, module_eval_order); - eprintln!("🐤 path {:?} | dependencies {:?}", relevant_path, dependencies); - - eprintln!("💾 Inserting cache entry for TY module {:?}", relevant_path); - + let modified_time = std::fs::metadata(path.as_path()) + .ok() + .and_then(|m| m.modified().ok()); let version = build_config .and_then(|config| config.lsp_mode.as_ref()) .and_then(|lsp| lsp.file_versions.get(&path).copied()) .flatten(); - engines - .qe() - .insert_ty_module_cache_entry(TyModuleCacheEntry { - path: Arc::new(path), + eprintln!("💾 Inserting cache entry for TY module {:?}", relevant_path); + let include_tests = build_config.map_or(false, |x| x.include_tests); + let key = ModuleCacheKey::new(path.clone().into(), include_tests); + engines.qe().update_typed_module_cache_entry( + &key, + TypedModuleInfo { module: ty_module.clone(), - dependencies, + modified_time, version, - }); + }, + ); } Ok(ty_module) diff --git a/sway-core/src/type_system/engine.rs b/sway-core/src/type_system/engine.rs index 9e2085bda39..2b91f499dc6 100644 --- a/sway-core/src/type_system/engine.rs +++ b/sway-core/src/type_system/engine.rs @@ -71,8 +71,11 @@ impl TypeEngine { where F: Fn(&SourceId) -> bool, { - self.slab.retain(|_, tsi| tsi.source_id.as_ref().map_or(true, &keep)); - self.id_map.write().retain(|tsi, _| tsi.source_id.as_ref().map_or(true, &keep)); + self.slab + .retain(|_, tsi| tsi.source_id.as_ref().map_or(true, &keep)); + self.id_map + .write() + .retain(|tsi, _| tsi.source_id.as_ref().map_or(true, &keep)); } /// Removes all data associated with `program_id` from the type engine. From f801b962f9aef8931cd3caa18466aeddd5f1a623 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Tue, 16 Jul 2024 16:25:59 +1000 Subject: [PATCH 08/21] wip --- sway-core/src/lib.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 24df5a46d75..73b248155cf 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -413,14 +413,6 @@ fn parse_module_tree( let version = lsp_mode .and_then(|lsp| lsp.file_versions.get(path.as_ref()).copied()) .unwrap_or(None); - // let cache_entry = ModuleCacheEntry { - // path: path.clone(), - // modified_time, - // hash, - // dependencies, - // include_tests, - // version, - // }; let common_info = ModuleCommonInfo { path: path.clone(), From aeaded0f18b2714cd5af9cc720413ceffb45c083 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Fri, 19 Jul 2024 10:35:59 +1000 Subject: [PATCH 09/21] more debugging start of friday --- forc-pkg/src/manifest/mod.rs | 4 +-- forc-pkg/src/pkg.rs | 1 + sway-core/src/lib.rs | 38 ++++++++++++++--------- sway-core/src/semantic_analysis/module.rs | 22 ++++++------- sway-lsp/src/core/session.rs | 7 ++++- sway-lsp/src/server_state.rs | 9 ++++++ sway-lsp/tests/lib.rs | 26 ++++++++++++---- 7 files changed, 73 insertions(+), 34 deletions(-) diff --git a/forc-pkg/src/manifest/mod.rs b/forc-pkg/src/manifest/mod.rs index 3f7bc21b1bd..ca35a46e9da 100644 --- a/forc-pkg/src/manifest/mod.rs +++ b/forc-pkg/src/manifest/mod.rs @@ -566,7 +566,7 @@ impl PackageManifest { }) .map_err(|e| anyhow!("failed to parse manifest: {}.", e))?; for warning in warnings { - println_warning(&warning); + //println_warning(&warning); } manifest.implicitly_include_std_if_missing(); manifest.implicitly_include_default_build_profiles_if_missing(); @@ -955,7 +955,7 @@ impl WorkspaceManifest { }) .map_err(|e| anyhow!("failed to parse manifest: {}.", e))?; for warning in warnings { - println_warning(&warning); + //println_warning(&warning); } Ok(manifest) } diff --git a/forc-pkg/src/pkg.rs b/forc-pkg/src/pkg.rs index 99da8d36402..598328f1ba5 100644 --- a/forc-pkg/src/pkg.rs +++ b/forc-pkg/src/pkg.rs @@ -2711,6 +2711,7 @@ pub fn check( .as_ref() .is_some_and(|b| b.load(std::sync::atomic::Ordering::SeqCst)) { + eprintln!("🪓 🪓 compilation was cancelled 2716"); bail!("compilation was retriggered") } diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 73b248155cf..a2381c68e15 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -409,7 +409,7 @@ fn parse_module_tree( .ok() .and_then(|m| m.modified().ok()); let dependencies = submodules.into_iter().map(|s| s.path).collect::>(); - eprintln!("🐔 path {:?} | dependencies {:?}", path, dependencies); + //eprintln!("🐔 path {:?} | dependencies {:?}", path, dependencies); let version = lsp_mode .and_then(|lsp| lsp.file_versions.get(path.as_ref()).copied()) .unwrap_or(None); @@ -468,7 +468,7 @@ pub(crate) fn is_ty_module_cache_up_to_date( ); if cache_up_to_date { entry.common.dependencies.iter().all(|path| { - eprint!("checking dep path {:?} ", path); + //eprint!("checking dep path {:?} ", path); is_ty_module_cache_up_to_date(engines, path, include_tests, build_config) }) } else { @@ -544,10 +544,18 @@ pub(crate) fn is_parse_module_cache_up_to_date( None => false, }; - eprintln!( - "🀄 👓 Checking cache for parse module {:?} | is up to date? {}", - relevant_path, res - ); + if res { + eprintln!( + "🀄 👓 Checking cache for parse module {:?} | is up to date? true 🟩", + relevant_path + ); + } else { + eprintln!( + "🀄 👓 Checking cache for parse module {:?} | is up to date? FALSE 🟥", + relevant_path + ); + } + res } @@ -618,7 +626,7 @@ pub fn parsed_to_ast( package_name, build_config, ); - check_should_abort(handler, retrigger_compilation.clone())?; + check_should_abort(handler, retrigger_compilation.clone(), 629)?; // Only clear the parsed AST nodes if we are running a regular compilation pipeline. // LSP needs these to build its token map, and they are cleared by `clear_program` as @@ -679,7 +687,7 @@ pub fn parsed_to_ast( None => (None, None), }; - check_should_abort(handler, retrigger_compilation.clone())?; + check_should_abort(handler, retrigger_compilation.clone(), 690)?; // Perform control flow analysis and extend with any errors. let _ = perform_control_flow_analysis( @@ -765,8 +773,8 @@ pub fn compile_to_ast( package_name: &str, retrigger_compilation: Option>, ) -> Result { - eprintln!("👨‍💻 compile_to_ast 👨‍💻"); - check_should_abort(handler, retrigger_compilation.clone())?; + eprintln!("🔨🔨 --- compile_to_ast --- 🔨🔨"); + check_should_abort(handler, retrigger_compilation.clone(), 777)?; let query_engine = engines.qe(); let mut metrics = PerformanceData::default(); if let Some(config) = build_config { @@ -794,7 +802,7 @@ pub fn compile_to_ast( metrics ); - check_should_abort(handler, retrigger_compilation.clone())?; + check_should_abort(handler, retrigger_compilation.clone(), 805)?; let (lexed_program, mut parsed_program) = match parse_program_opt { Ok(modules) => modules, @@ -809,7 +817,7 @@ pub fn compile_to_ast( parsed_program.exclude_tests(engines); } - eprintln!("👩‍💻 parsed to typed AST 👩‍💻"); + eprintln!("🔨🔨 --- parsed to typed AST 🔨🔨 ---"); // Type check (+ other static analysis) the CST to a typed AST. let typed_res = time_expr!( "parse the concrete syntax tree (CST) to a typed AST", @@ -827,7 +835,7 @@ pub fn compile_to_ast( metrics ); - check_should_abort(handler, retrigger_compilation.clone())?; + check_should_abort(handler, retrigger_compilation.clone(), 838)?; handler.dedup(); @@ -843,7 +851,7 @@ pub fn compile_to_ast( query_engine.insert_programs_cache_entry(cache_entry); } - check_should_abort(handler, retrigger_compilation.clone())?; + check_should_abort(handler, retrigger_compilation.clone(), 854)?; Ok(programs) } @@ -1150,9 +1158,11 @@ fn module_return_path_analysis( fn check_should_abort( handler: &Handler, retrigger_compilation: Option>, + line_num: u32, ) -> Result<(), ErrorEmitted> { if let Some(ref retrigger_compilation) = retrigger_compilation { if retrigger_compilation.load(Ordering::SeqCst) { + eprintln!("🪓 🪓 compilation was cancelled at line {} 🪓 🪓", line_num); return Err(handler.cancel()); } } diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 26131b08eef..7a22fc2e886 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -293,8 +293,8 @@ impl ty::TyModule { build_config, ) { eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? {}", - relevant_path, true + "📟 👓 Checking cache for TY module {:?} | is up to date? true 🟩", + relevant_path ); eprintln!("✅ Cache hit for module {:?}", relevant_path); @@ -302,11 +302,11 @@ impl ty::TyModule { return Some(typed.module); } else { eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? {}", - relevant_path, false + "📟 👓 Checking cache for TY module {:?} | is up to date? false 🟥", + relevant_path ); - eprintln!("🔄 Cache unable to be used for module {:?}", relevant_path); + eprintln!("🔄 🟨 🟨 🟨 Cache unable to be used for module {:?}", relevant_path); } } } @@ -371,12 +371,12 @@ impl ty::TyModule { }) .collect::, _>>(); - // Check if the root module cache is up to date - if let Some(module) = - ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) - { - return Ok(module); - } + // // Check if the root module cache is up to date + // if let Some(module) = + // ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) + // { + // return Ok(module); + // } // TODO: Ordering should be solved across all modules prior to the beginning of type-check. let ordered_nodes = node_dependencies::order_ast_nodes_by_dependency( diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index 0236e278051..2fcdbe8643d 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -143,7 +143,7 @@ impl Session { uri: &Url, ) -> Result<(), LanguageServerError> { let path = uri.to_file_path().unwrap(); - eprintln!("🗑️ Garbage collecting module {:?}", path); + eprintln!("🗑️ 🗑️ 🗑️ 🗑️ 🗑️ Garbage collecting module {:?} 🗑️ 🗑️ 🗑️ 🗑️ 🗑️", path); let source_id = { engines.se().get_source_id(&path) }; engines.clear_module(&source_id); Ok(()) @@ -338,6 +338,8 @@ pub fn traverse( let path = engines.se().get_path(source_id); let program_id = program_id_from_path(&path, engines)?; session.metrics.insert(program_id, metrics); + + eprintln!("⤵️ Traversing: {:?}", path); } // Get a reference to the typed program AST. @@ -412,6 +414,7 @@ pub fn parse_project( if results.last().is_none() { return Err(LanguageServerError::ProgramsIsNone); } + eprintln!("⤵️ Traversing the ASTS"); let diagnostics = traverse(results, engines, session.clone())?; if let Some(config) = &lsp_mode { // Only write the diagnostics results on didSave or didOpen. @@ -435,6 +438,7 @@ fn parse_ast_to_tokens( ctx: &ParseContext, f: impl Fn(&AstNode, &ParseContext) + Sync, ) { + eprintln!("⤵️ Parsing the AST"); let nodes = parse_program .root .tree @@ -456,6 +460,7 @@ fn parse_ast_to_typed_tokens( ctx: &ParseContext, f: impl Fn(&ty::TyAstNode, &ParseContext) + Sync, ) { + eprintln!("⤵️ Parsing the typed AST"); let nodes = typed_program .root .all_nodes diff --git a/sway-lsp/src/server_state.rs b/sway-lsp/src/server_state.rs index 57ab7cbf0d3..a35c713888e 100644 --- a/sway-lsp/src/server_state.rs +++ b/sway-lsp/src/server_state.rs @@ -132,6 +132,7 @@ impl ServerState { let uri = ctx.uri.as_ref().unwrap().clone(); let session = ctx.session.as_ref().unwrap().clone(); let mut engines_clone = session.engines.read().clone(); + eprintln!("\n ----------------- NEW COMPILATION TASK: triggered by {:?} -----------------", uri.path()); if let Some(version) = ctx.version { // Perform garbage collection at configured intervals if enabled to manage memory usage. @@ -149,6 +150,8 @@ impl ServerState { err.to_string() ); } + } else { + eprintln!("No Garbabe collection applied"); } } @@ -159,6 +162,7 @@ impl ServerState { // Set the is_compiling flag to true so that the wait_for_parsing function knows that we are compiling is_compiling.store(true, Ordering::SeqCst); + eprintln!("⚙️ ⚙️ ⚙️ ⚙️ session::parse_project ⚙️ ⚙️ ⚙️ ⚙️"); match session::parse_project( &uri, &engines_clone, @@ -172,14 +176,17 @@ impl ServerState { // Find the module id from the path match session::program_id_from_path(&path, &engines_clone) { Ok(program_id) => { + eprintln!("👨‍💻 ✅ Compliation returned successfully 👨‍💻"); // Use the module id to get the metrics for the module if let Some(metrics) = session.metrics.get(&program_id) { // It's very important to check if the workspace AST was reused to determine if we need to overwrite the engines. // Because the engines_clone has garbage collection applied. If the workspace AST was reused, we need to keep the old engines // as the engines_clone might have cleared some types that are still in use. + eprintln!("👨‍💻 metrics.reused_programs {} 👨‍💻", metrics.reused_programs); if metrics.reused_programs == 0 { // The compiler did not reuse the workspace AST. // We need to overwrite the old engines with the engines clone. + eprintln!("👨‍💻 ↪ Swapping engines 👨‍💻 ↪"); mem::swap( &mut *session.engines.write(), &mut engines_clone, @@ -190,6 +197,7 @@ impl ServerState { LastCompilationState::Success; } Err(err) => { + eprintln!("👨‍💻 ❌ Compliation failed 👨‍💻"); tracing::error!("{}", err.to_string()); *last_compilation_state.write() = LastCompilationState::Failed; @@ -197,6 +205,7 @@ impl ServerState { } } Err(_err) => { + eprintln!("👨‍💻 ❌ Compliation failed 👨‍💻"); *last_compilation_state.write() = LastCompilationState::Failed; } } diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index 9f3faeb974d..aaa5f20526b 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -233,15 +233,29 @@ fn did_change_stress_test() { #[test] fn did_change_stress_test_random_wait() { run_async!({ - let test_duration = tokio::time::Duration::from_secs(5 * 60); // 5 minutes timeout + let test_duration = tokio::time::Duration::from_secs(250 * 60); // 5 minutes timeout let test_future = async { setup_panic_hook(); let (mut service, _) = LspService::new(ServerState::new); - let example_dir = sway_workspace_dir() - .join(e2e_language_dir()) - .join("generics_in_contract"); - let uri = init_and_open(&mut service, example_dir.join("src/main.sw")).await; - let times = 60; + // let example_dir = sway_workspace_dir() + // .join(e2e_language_dir()) + // .join("generics_in_contract"); +// let uri = init_and_open(&mut service, example_dir.join("src/main.sw")).await; + + let uri = init_and_open( + &mut service, + PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries") + .join("src/fpt_staking_interface.sw"), + ) + .await; + + // 1. randomise the file that is changed out of all the files in the project. + // 2. change the file + // 3. wait for the file to be parsed + // 4. try and do a hover or goto def for a random type + // 5. repeat. + + let times = 6000; for version in 0..times { //eprintln!("version: {}", version); let _ = lsp::did_change_request(&mut service, &uri, version + 1, None).await; From 122d98ebacc164de02f58b4914e595d2d3933e87 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Fri, 19 Jul 2024 11:28:50 +1000 Subject: [PATCH 10/21] check if root module is up to date --- sway-core/src/semantic_analysis/module.rs | 15 +++++++++------ .../src/semantic_analysis/node_dependencies.rs | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 7a22fc2e886..a9a76539bb7 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -334,6 +334,14 @@ impl ty::TyModule { .. } = parsed; + // Check if the root module cache is up to date + eprintln!("Root Module: {:?}", parsed.span.source_id().map(|x| engines.se().get_path(x))); + if let Some(module) = + ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) + { + return Ok(module); + } + // Type-check submodules first in order of evaluation previously computed by the dependency graph. let submodules_res = module_eval_order .iter() @@ -371,12 +379,7 @@ impl ty::TyModule { }) .collect::, _>>(); - // // Check if the root module cache is up to date - // if let Some(module) = - // ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) - // { - // return Ok(module); - // } + // TODO: Ordering should be solved across all modules prior to the beginning of type-check. let ordered_nodes = node_dependencies::order_ast_nodes_by_dependency( diff --git a/sway-core/src/semantic_analysis/node_dependencies.rs b/sway-core/src/semantic_analysis/node_dependencies.rs index 45d2858a1b8..9edf1cde31f 100644 --- a/sway-core/src/semantic_analysis/node_dependencies.rs +++ b/sway-core/src/semantic_analysis/node_dependencies.rs @@ -42,7 +42,7 @@ pub(crate) fn order_ast_nodes_by_dependency( Ok(()) })?; - // Reorder the parsed AstNodes based on dependency. Includes first, then uses, then + // Reorder the parsed AstNodes based on dependency. Includes first, then uses, then // reordered declarations, then anything else. To keep the list stable and simple we can // use a basic insertion sort. Ok(nodes From 5d566a71b7e389089d94e996b656dec1800f11b5 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Fri, 19 Jul 2024 12:14:57 +1000 Subject: [PATCH 11/21] add debug timings --- forc-pkg/src/pkg.rs | 2 ++ sway-lsp/src/core/session.rs | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/forc-pkg/src/pkg.rs b/forc-pkg/src/pkg.rs index 598328f1ba5..be59f93db7c 100644 --- a/forc-pkg/src/pkg.rs +++ b/forc-pkg/src/pkg.rs @@ -2655,6 +2655,7 @@ pub fn check( let mut results = vec![]; for (idx, &node) in plan.compilation_order.iter().enumerate() { + let now = std::time::Instant::now(); let pkg = &plan.graph[node]; let manifest = &plan.manifest_map()[&pkg.id()]; @@ -2747,6 +2748,7 @@ pub fn check( return Ok(results); } results.push((programs_res.ok(), handler)); + eprintln!("⏱️ Compiling package {:?} took {:?}", pkg.name, now.elapsed()); } if results.is_empty() { diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index 2fcdbe8643d..8ff0d293f03 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -26,6 +26,7 @@ use pkg::{ BuildPlan, }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use swayfmt::parse; use std::{ path::PathBuf, sync::{atomic::AtomicBool, Arc}, @@ -279,6 +280,9 @@ pub fn compile( experimental: sway_core::ExperimentalFlags, ) -> Result, Handler)>, LanguageServerError> { let _p = tracing::trace_span!("compile").entered(); + let build_plan_now = std::time::Instant::now(); + let build_plan = build_plan(uri)?; + eprintln!("⏱️ Build Plan took {:?}", build_plan_now.elapsed()); let tests_enabled = true; pkg::check( build_plan, @@ -404,6 +408,7 @@ pub fn parse_project( let build_plan = session .build_plan_cache .get_or_update(&session.sync.manifest_path(), || build_plan(uri))?; + let parse_now = std::time::Instant::now(); let results = compile( &build_plan, engines, @@ -411,11 +416,14 @@ pub fn parse_project( lsp_mode.clone(), experimental, )?; + eprintln!("⏱️ Total Compilation took {:?}", parse_now.elapsed()); if results.last().is_none() { return Err(LanguageServerError::ProgramsIsNone); } eprintln!("⤵️ Traversing the ASTS"); + let traverse_now = std::time::Instant::now(); let diagnostics = traverse(results, engines, session.clone())?; + eprintln!("⏱️ Traversing the ASTS took {:?}", traverse_now.elapsed()); if let Some(config) = &lsp_mode { // Only write the diagnostics results on didSave or didOpen. if !config.optimized_build { @@ -425,10 +433,13 @@ pub fn parse_project( } } } + let runnables_now = std::time::Instant::now(); if let Some(typed) = &session.compiled_program.read().typed { session.runnables.clear(); create_runnables(&session.runnables, typed, engines.de(), engines.se()); } + eprintln!("⏱️ creating runnables took: {:?}", runnables_now.elapsed()); + eprintln!("⏱️ TOTAL COMPILATION AND TRAVERSAL TIME: {:?}", parse_now.elapsed()); Ok(()) } From 2f905b5aa8d1bb24316c7a9e030dad6516c5b35d Mon Sep 17 00:00:00 2001 From: Joshua Batty Date: Mon, 22 Jul 2024 12:10:22 +1000 Subject: [PATCH 12/21] more timings --- forc-pkg/src/pkg.rs | 6 ++++++ sway-core/Cargo.toml | 1 + sway-core/src/lib.rs | 11 ++++++++++- sway-lsp/src/core/session.rs | 2 ++ sway-lsp/tests/lib.rs | 2 +- 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/forc-pkg/src/pkg.rs b/forc-pkg/src/pkg.rs index be59f93db7c..e515cf339a6 100644 --- a/forc-pkg/src/pkg.rs +++ b/forc-pkg/src/pkg.rs @@ -2671,6 +2671,7 @@ pub fn check( let contract_id_value = (idx == plan.compilation_order.len() - 1).then(|| DUMMY_CONTRACT_ID.to_string()); + let dep_now = std::time::Instant::now(); let mut dep_namespace = dependency_namespace( &lib_namespace_map, &compiled_contract_deps, @@ -2681,12 +2682,14 @@ pub fn check( experimental, ) .expect("failed to create dependency namespace"); + eprintln!("⏱️ Dependency namespace took {:?}", dep_now.elapsed()); let profile = BuildProfile { terse: terse_mode, ..BuildProfile::debug() }; + let build_config_now = std::time::Instant::now(); let build_config = sway_build_config( manifest.dir(), &manifest.entry_path(), @@ -2695,9 +2698,11 @@ pub fn check( )? .with_include_tests(include_tests) .with_lsp_mode(lsp_mode.clone()); + eprintln!("⏱️ Build config took {:?}", build_config_now.elapsed()); let input = manifest.entry_string()?; let handler = Handler::default(); + let compile_to_ast_now = std::time::Instant::now(); let programs_res = sway_core::compile_to_ast( &handler, engines, @@ -2707,6 +2712,7 @@ pub fn check( &pkg.name, retrigger_compilation.clone(), ); + eprintln!("⏱️ Compile to AST took {:?}", compile_to_ast_now.elapsed()); if retrigger_compilation .as_ref() diff --git a/sway-core/Cargo.toml b/sway-core/Cargo.toml index 9c0122123c2..a903dca0b7b 100644 --- a/sway-core/Cargo.toml +++ b/sway-core/Cargo.toml @@ -34,6 +34,7 @@ parking_lot = "0.12" pest = "2.1.3" pest_derive = "2.1" petgraph = "0.6" +rayon = "1.5" rustc-hash = "1.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.91" diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index a2381c68e15..9be80676d8b 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -31,10 +31,12 @@ pub use asm_generation::{CompiledBytecode, FinalizedEntry}; pub use build_config::{BuildConfig, BuildTarget, LspConfig, OptLevel, PrintAsm, PrintIr}; use control_flow_analysis::ControlFlowGraph; pub use debug_generation::write_dwarf; +use decl_engine::parsed_engine; use fuel_vm::fuel_merkle::common; use indexmap::IndexMap; use metadata::MetadataManager; use query_engine::{ModuleCacheKey, ModuleCommonInfo, ParsedModuleInfo, ProgramsCacheEntry}; +use rayon::iter::{ParallelBridge, ParallelIterator}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; @@ -348,6 +350,7 @@ fn parse_module_tree( ) -> Result { let query_engine = engines.qe(); + let lexed_now = std::time::Instant::now(); // Parse this module first. let module_dir = path.parent().expect("module file has no parent directory"); let source_id = engines.se().get_source_id(&path.clone()); @@ -366,7 +369,9 @@ fn parse_module_tree( experimental, lsp_mode, ); + eprintln!("⏱️ Lexed module took {:?}", lexed_now.elapsed()); + let parsed_now = std::time::Instant::now(); // Convert from the raw parsed module to the `ParseTree` ready for type-check. let (kind, tree) = to_parsed_lang::convert_parse_tree( &mut to_parsed_lang::Context::new(build_target, experimental), @@ -376,6 +381,7 @@ fn parse_module_tree( )?; let module_kind_span = module.value.kind.span(); let attributes = module_attrs_to_map(handler, &module.attribute_list)?; + eprintln!("⏱️ Parsed module took {:?}", parsed_now.elapsed()); let lexed_submodules = submodules .iter() @@ -555,7 +561,6 @@ pub(crate) fn is_parse_module_cache_up_to_date( relevant_path ); } - res } @@ -775,12 +780,16 @@ pub fn compile_to_ast( ) -> Result { eprintln!("🔨🔨 --- compile_to_ast --- 🔨🔨"); check_should_abort(handler, retrigger_compilation.clone(), 777)?; + let query_engine = engines.qe(); let mut metrics = PerformanceData::default(); if let Some(config) = build_config { let path = config.canonical_root_module(); let include_tests = config.include_tests; + //eprintln!(" 📂 {}", path.display()); + //eprintln!("{:?}", config.lsp_mode.as_ref().unwrap().file_versions); + // Check if we can re-use the data in the cache. if is_parse_module_cache_up_to_date(engines, &path, include_tests, build_config) { let mut entry = query_engine.get_programs_cache_entry(&path).unwrap(); diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index 8ff0d293f03..ab58688acee 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -310,6 +310,7 @@ pub fn traverse( let mut diagnostics: CompileResults = (Vec::default(), Vec::default()); let results_len = results.len(); for (i, (value, handler)) in results.into_iter().enumerate() { + let parse_now = std::time::Instant::now(); // We can convert these destructured elements to a Vec later on. let current_diagnostics = handler.consume(); diagnostics = current_diagnostics; @@ -391,6 +392,7 @@ pub fn traverse( dependency::collect_typed_declaration(node, ctx); }); } + eprintln!("⏱️ Traversal took {:?}", parse_now.elapsed()); } Ok(Some(diagnostics)) } diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index aaa5f20526b..bcce174ef0d 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -244,7 +244,7 @@ fn did_change_stress_test_random_wait() { let uri = init_and_open( &mut service, - PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries") + PathBuf::from("/Users/joshuabatty/Documents/rust/fuel/user_projects/fluid-protocol/libraries") .join("src/fpt_staking_interface.sw"), ) .await; From 5c98e15f035d0ba4eed9ff5a546bc406413c7299 Mon Sep 17 00:00:00 2001 From: Joshua Batty Date: Mon, 22 Jul 2024 13:35:52 +1000 Subject: [PATCH 13/21] clean up --- sway-core/src/semantic_analysis/module.rs | 28 +++++++++-------------- sway-lsp/src/handlers/notification.rs | 1 + sway-lsp/tests/lib.rs | 2 +- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index a9a76539bb7..eb3278e37ed 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -273,7 +273,6 @@ impl ty::TyModule { .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - eprintln!("🥸 Checking cache for TY module {:?}", relevant_path); let key: ModuleCacheKey = ModuleCacheKey::new(path.clone().into(), include_tests); if let Some(entry) = engines.qe().get_module_cache_entry(&key) { @@ -286,29 +285,24 @@ impl ty::TyModule { // Let's check if we can re-use the dependency information // we got from the cache. if let Some(typed) = entry.typed { - if is_ty_module_cache_up_to_date( + let is_up_to_date = is_ty_module_cache_up_to_date( engines, &path.into(), include_tests, build_config, - ) { - eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? true 🟩", - relevant_path - ); - - eprintln!("✅ Cache hit for module {:?}", relevant_path); - // Return the cached module - return Some(typed.module); + ); + let status = if is_up_to_date { + "✅ Cache hit" } else { - eprintln!( - "📟 👓 Checking cache for TY module {:?} | is up to date? false 🟥", - relevant_path - ); - - eprintln!("🔄 🟨 🟨 🟨 Cache unable to be used for module {:?}", relevant_path); + "🔄 Cache miss" + }; + eprintln!("{} for module {:?} (up to date: {})", status, relevant_path, is_up_to_date); + if is_up_to_date { + return Some(typed.module); } } + } else { + eprintln!("❌ No cache entry for module {:?}", relevant_path); } } None diff --git a/sway-lsp/src/handlers/notification.rs b/sway-lsp/src/handlers/notification.rs index 99cb0406bef..d62be57c69c 100644 --- a/sway-lsp/src/handlers/notification.rs +++ b/sway-lsp/src/handlers/notification.rs @@ -106,6 +106,7 @@ pub async fn handle_did_change_text_document( &uri, Some(params.text_document.version as u64), ); + eprintln!("File versions: {:#?}", file_versions); send_new_compilation_request( state, session.clone(), diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index bcce174ef0d..adad70db71e 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -171,7 +171,7 @@ fn did_open_fluid_libraries() { .finish(); let uri = init_and_open( &mut service, - PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries") + PathBuf::from("/Users/joshuabatty/Documents/rust/fuel/user_projects/fluid-protocol/libraries") .join("src/interface.sw"), ) .await; From 9875b7c81c03d3aa9da7cf63cd87a1d73aa3c588 Mon Sep 17 00:00:00 2001 From: Joshua Batty Date: Tue, 23 Jul 2024 11:08:58 +1000 Subject: [PATCH 14/21] wip witching to mac studio --- sway-lsp/src/handlers/notification.rs | 2 +- sway-lsp/src/server_state.rs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sway-lsp/src/handlers/notification.rs b/sway-lsp/src/handlers/notification.rs index d62be57c69c..68c35a99802 100644 --- a/sway-lsp/src/handlers/notification.rs +++ b/sway-lsp/src/handlers/notification.rs @@ -106,7 +106,7 @@ pub async fn handle_did_change_text_document( &uri, Some(params.text_document.version as u64), ); - eprintln!("File versions: {:#?}", file_versions); + //eprintln!("File versions: {:#?}", file_versions); send_new_compilation_request( state, session.clone(), diff --git a/sway-lsp/src/server_state.rs b/sway-lsp/src/server_state.rs index a35c713888e..a26724f8cd1 100644 --- a/sway-lsp/src/server_state.rs +++ b/sway-lsp/src/server_state.rs @@ -379,6 +379,9 @@ impl ServerState { let session = self.sessions.get(&manifest_dir).unwrap_or({ // If no session can be found, then we need to call init and insert a new session into the map + eprintln!("💥💥💥💥💥💥💥💥💥💥💥💥💥💥💥 Initializing new session for manifest_dir = {:?}", manifest_dir); + + self.init_session(uri).await?; self.sessions .get(&manifest_dir) From 1ea636b89bd04e3163c2192121b37234e4968d5b Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Tue, 23 Jul 2024 11:59:30 +1000 Subject: [PATCH 15/21] fix session cache bug --- forc-pkg/src/pkg.rs | 6 +- sway-core/src/semantic_analysis/module.rs | 12 ++- sway-lsp/src/core/session.rs | 21 ++-- sway-lsp/src/server_state.rs | 124 ++++++++++++++++++---- sway-lsp/tests/lib.rs | 55 ++++++++-- 5 files changed, 172 insertions(+), 46 deletions(-) diff --git a/forc-pkg/src/pkg.rs b/forc-pkg/src/pkg.rs index e515cf339a6..cbd38f05359 100644 --- a/forc-pkg/src/pkg.rs +++ b/forc-pkg/src/pkg.rs @@ -2754,7 +2754,11 @@ pub fn check( return Ok(results); } results.push((programs_res.ok(), handler)); - eprintln!("⏱️ Compiling package {:?} took {:?}", pkg.name, now.elapsed()); + eprintln!( + "⏱️ Compiling package {:?} took {:?}", + pkg.name, + now.elapsed() + ); } if results.is_empty() { diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index eb3278e37ed..a6d71b519db 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -296,7 +296,10 @@ impl ty::TyModule { } else { "🔄 Cache miss" }; - eprintln!("{} for module {:?} (up to date: {})", status, relevant_path, is_up_to_date); + eprintln!( + "{} for module {:?} (up to date: {})", + status, relevant_path, is_up_to_date + ); if is_up_to_date { return Some(typed.module); } @@ -329,7 +332,10 @@ impl ty::TyModule { } = parsed; // Check if the root module cache is up to date - eprintln!("Root Module: {:?}", parsed.span.source_id().map(|x| engines.se().get_path(x))); + eprintln!( + "Root Module: {:?}", + parsed.span.source_id().map(|x| engines.se().get_path(x)) + ); if let Some(module) = ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) { @@ -373,8 +379,6 @@ impl ty::TyModule { }) .collect::, _>>(); - - // TODO: Ordering should be solved across all modules prior to the beginning of type-check. let ordered_nodes = node_dependencies::order_ast_nodes_by_dependency( handler, diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index ab58688acee..a25449bc43a 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -26,7 +26,6 @@ use pkg::{ BuildPlan, }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use swayfmt::parse; use std::{ path::PathBuf, sync::{atomic::AtomicBool, Arc}, @@ -44,6 +43,7 @@ use sway_core::{ use sway_error::{error::CompileError, handler::Handler, warning::CompileWarning}; use sway_types::{ProgramId, SourceEngine, Spanned}; use sway_utils::{helpers::get_sway_files, PerformanceData}; +use swayfmt::parse; pub type RunnableMap = DashMap>>; pub type ProjectDirectory = PathBuf; @@ -144,7 +144,10 @@ impl Session { uri: &Url, ) -> Result<(), LanguageServerError> { let path = uri.to_file_path().unwrap(); - eprintln!("🗑️ 🗑️ 🗑️ 🗑️ 🗑️ Garbage collecting module {:?} 🗑️ 🗑️ 🗑️ 🗑️ 🗑️", path); + eprintln!( + "🗑️ 🗑️ 🗑️ 🗑️ 🗑️ Garbage collecting module {:?} 🗑️ 🗑️ 🗑️ 🗑️ 🗑️", + path + ); let source_id = { engines.se().get_source_id(&path) }; engines.clear_module(&source_id); Ok(()) @@ -280,16 +283,12 @@ pub fn compile( experimental: sway_core::ExperimentalFlags, ) -> Result, Handler)>, LanguageServerError> { let _p = tracing::trace_span!("compile").entered(); - let build_plan_now = std::time::Instant::now(); - let build_plan = build_plan(uri)?; - eprintln!("⏱️ Build Plan took {:?}", build_plan_now.elapsed()); - let tests_enabled = true; pkg::check( build_plan, BuildTarget::default(), true, lsp_mode, - tests_enabled, + true, engines, retrigger_compilation, experimental, @@ -407,9 +406,12 @@ pub fn parse_project( experimental: sway_core::ExperimentalFlags, ) -> Result<(), LanguageServerError> { let _p = tracing::trace_span!("parse_project").entered(); + let build_plan_now = std::time::Instant::now(); let build_plan = session .build_plan_cache .get_or_update(&session.sync.manifest_path(), || build_plan(uri))?; + eprintln!("⏱️ Build Plan took {:?}", build_plan_now.elapsed()); + let parse_now = std::time::Instant::now(); let results = compile( &build_plan, @@ -441,7 +443,10 @@ pub fn parse_project( create_runnables(&session.runnables, typed, engines.de(), engines.se()); } eprintln!("⏱️ creating runnables took: {:?}", runnables_now.elapsed()); - eprintln!("⏱️ TOTAL COMPILATION AND TRAVERSAL TIME: {:?}", parse_now.elapsed()); + eprintln!( + "⏱️ TOTAL COMPILATION AND TRAVERSAL TIME: {:?}", + parse_now.elapsed() + ); Ok(()) } diff --git a/sway-lsp/src/server_state.rs b/sway-lsp/src/server_state.rs index a26724f8cd1..a65dc51a6d7 100644 --- a/sway-lsp/src/server_state.rs +++ b/sway-lsp/src/server_state.rs @@ -41,7 +41,7 @@ pub struct ServerState { /// A Least Recently Used (LRU) cache of [Session]s, each representing a project opened in the user's workspace. /// This cache limits memory usage by maintaining a fixed number of active sessions, automatically /// evicting the least recently used sessions when the capacity is reached. - pub(crate) sessions: LruSessionCache, + pub sessions: LruSessionCache, pub documents: Documents, // Compilation thread related fields pub(crate) retrigger_compilation: Arc, @@ -182,7 +182,10 @@ impl ServerState { // It's very important to check if the workspace AST was reused to determine if we need to overwrite the engines. // Because the engines_clone has garbage collection applied. If the workspace AST was reused, we need to keep the old engines // as the engines_clone might have cleared some types that are still in use. - eprintln!("👨‍💻 metrics.reused_programs {} 👨‍💻", metrics.reused_programs); + eprintln!( + "👨‍💻 metrics.reused_programs {} 👨‍💻", + metrics.reused_programs + ); if metrics.reused_programs == 0 { // The compiler did not reuse the workspace AST. // We need to overwrite the old engines with the engines clone. @@ -353,7 +356,7 @@ impl ServerState { /// Constructs and returns a tuple of `(Url, Arc)` from a given workspace URI. /// The returned URL represents the temp directory workspace. - pub(crate) async fn uri_and_session_from_workspace( + pub async fn uri_and_session_from_workspace( &self, workspace_uri: &Url, ) -> Result<(Url, Arc), LanguageServerError> { @@ -377,23 +380,23 @@ impl ServerState { .ok_or(DirectoryError::ManifestDirNotFound)? .to_path_buf(); - let session = self.sessions.get(&manifest_dir).unwrap_or({ - // If no session can be found, then we need to call init and insert a new session into the map - eprintln!("💥💥💥💥💥💥💥💥💥💥💥💥💥💥💥 Initializing new session for manifest_dir = {:?}", manifest_dir); + // If the session is already in the cache, return it + if let Some(session) = self.sessions.get(&manifest_dir) { + return Ok(session); + } + // If no session can be found, then we need to call init and insert a new session into the map + let session = Arc::new(Session::new()); + session.init(uri, &self.documents).await?; + self.sessions.insert(manifest_dir.clone(), session.clone()); - self.init_session(uri).await?; - self.sessions - .get(&manifest_dir) - .expect("no session found even though it was just inserted into the map") - }); Ok(session) } } /// A Least Recently Used (LRU) cache for storing and managing `Session` objects. /// This cache helps limit memory usage by maintaining a fixed number of active sessions. -pub(crate) struct LruSessionCache { +pub struct LruSessionCache { /// Stores the actual `Session` objects, keyed by their file paths. sessions: Arc>>, /// Keeps track of the order in which sessions were accessed, with most recent at the front. @@ -404,7 +407,7 @@ pub(crate) struct LruSessionCache { impl LruSessionCache { /// Creates a new `LruSessionCache` with the specified capacity. - pub(crate) fn new(capacity: usize) -> Self { + pub fn new(capacity: usize) -> Self { LruSessionCache { sessions: Arc::new(DashMap::new()), usage_order: Arc::new(Mutex::new(VecDeque::with_capacity(capacity))), @@ -412,12 +415,12 @@ impl LruSessionCache { } } - pub(crate) fn iter(&self) -> impl Iterator>> { + pub fn iter(&self) -> impl Iterator>> { self.sessions.iter() } /// Retrieves a session from the cache and updates its position to the front of the usage order. - pub(crate) fn get(&self, path: &PathBuf) -> Option> { + pub fn get(&self, path: &PathBuf) -> Option> { if let Some(session) = self.sessions.try_get(path).try_unwrap() { if self.sessions.len() >= self.capacity { self.move_to_front(path); @@ -431,16 +434,13 @@ impl LruSessionCache { /// Inserts or updates a session in the cache. /// If at capacity and inserting a new session, evicts the least recently used one. /// For existing sessions, updates their position in the usage order if at capacity. - pub(crate) fn insert(&self, path: PathBuf, session: Arc) { - if self.sessions.get(&path).is_some() { - tracing::trace!("Updating existing session for path: {:?}", path); - // Session already exists, just update its position in the usage order if at capacity - if self.sessions.len() >= self.capacity { - self.move_to_front(&path); - } + pub fn insert(&self, path: PathBuf, session: Arc) { + if let Some(mut entry) = self.sessions.get_mut(&path) { + // Session already exists, update it + *entry = session; + self.move_to_front(&path); } else { // New session - tracing::trace!("Inserting new session for path: {:?}", path); if self.sessions.len() >= self.capacity { self.evict_least_used(); } @@ -472,3 +472,81 @@ impl LruSessionCache { } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::sync::Arc; + + #[test] + fn test_lru_session_cache_insertion_and_retrieval() { + let cache = LruSessionCache::new(2); + let path1 = PathBuf::from("/path/1"); + let path2 = PathBuf::from("/path/2"); + let session1 = Arc::new(Session::new()); + let session2 = Arc::new(Session::new()); + + cache.insert(path1.clone(), session1.clone()); + cache.insert(path2.clone(), session2.clone()); + + assert!(Arc::ptr_eq(&cache.get(&path1).unwrap(), &session1)); + assert!(Arc::ptr_eq(&cache.get(&path2).unwrap(), &session2)); + } + + #[test] + fn test_lru_session_cache_capacity() { + let cache = LruSessionCache::new(2); + let path1 = PathBuf::from("/path/1"); + let path2 = PathBuf::from("/path/2"); + let path3 = PathBuf::from("/path/3"); + let session1 = Arc::new(Session::new()); + let session2 = Arc::new(Session::new()); + let session3 = Arc::new(Session::new()); + + cache.insert(path1.clone(), session1); + cache.insert(path2.clone(), session2); + cache.insert(path3.clone(), session3); + + assert!(cache.get(&path1).is_none()); + assert!(cache.get(&path2).is_some()); + assert!(cache.get(&path3).is_some()); + } + + #[test] + fn test_lru_session_cache_update_order() { + let cache = LruSessionCache::new(2); + let path1 = PathBuf::from("/path/1"); + let path2 = PathBuf::from("/path/2"); + let path3 = PathBuf::from("/path/3"); + let session1 = Arc::new(Session::new()); + let session2 = Arc::new(Session::new()); + let session3 = Arc::new(Session::new()); + + cache.insert(path1.clone(), session1.clone()); + cache.insert(path2.clone(), session2.clone()); + + // Access path1 to move it to the front + cache.get(&path1); + + // Insert path3, which should evict path2 + cache.insert(path3.clone(), session3); + + assert!(cache.get(&path1).is_some()); + assert!(cache.get(&path2).is_none()); + assert!(cache.get(&path3).is_some()); + } + + #[test] + fn test_lru_session_cache_overwrite() { + let cache = LruSessionCache::new(2); + let path1 = PathBuf::from("/path/1"); + let session1 = Arc::new(Session::new()); + let session1_new = Arc::new(Session::new()); + + cache.insert(path1.clone(), session1); + cache.insert(path1.clone(), session1_new.clone()); + + assert!(Arc::ptr_eq(&cache.get(&path1).unwrap(), &session1_new)); + } +} diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index adad70db71e..acdb22ffe3b 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -171,7 +171,7 @@ fn did_open_fluid_libraries() { .finish(); let uri = init_and_open( &mut service, - PathBuf::from("/Users/joshuabatty/Documents/rust/fuel/user_projects/fluid-protocol/libraries") + PathBuf::from("/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries") .join("src/interface.sw"), ) .await; @@ -239,21 +239,23 @@ fn did_change_stress_test_random_wait() { let (mut service, _) = LspService::new(ServerState::new); // let example_dir = sway_workspace_dir() // .join(e2e_language_dir()) - // .join("generics_in_contract"); -// let uri = init_and_open(&mut service, example_dir.join("src/main.sw")).await; + // .join("generics_in_contract"); + // let uri = init_and_open(&mut service, example_dir.join("src/main.sw")).await; let uri = init_and_open( &mut service, - PathBuf::from("/Users/joshuabatty/Documents/rust/fuel/user_projects/fluid-protocol/libraries") - .join("src/fpt_staking_interface.sw"), + PathBuf::from( + "/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries", + ) + .join("src/fpt_staking_interface.sw"), ) .await; - // 1. randomise the file that is changed out of all the files in the project. - // 2. change the file - // 3. wait for the file to be parsed - // 4. try and do a hover or goto def for a random type - // 5. repeat. + // 1. randomise the file that is changed out of all the files in the project. + // 2. change the file + // 3. wait for the file to be parsed + // 4. try and do a hover or goto def for a random type + // 5. repeat. let times = 6000; for version in 0..times { @@ -2191,3 +2193,36 @@ async fn write_all_example_asts() { } let _ = server.shutdown_server(); } + +#[test] +fn test_url_to_session_existing_session() { + use std::sync::Arc; + run_async!({ + let (mut service, _) = LspService::new(ServerState::new); + let uri = init_and_open(&mut service, doc_comments_dir().join("src/main.sw")).await; + + // First call to uri_and_session_from_workspace + let (first_uri, first_session) = service + .inner() + .uri_and_session_from_workspace(&uri) + .await + .unwrap(); + + // Second call to uri_and_session_from_workspace + let (second_uri, second_session) = service + .inner() + .uri_and_session_from_workspace(&uri) + .await + .unwrap(); + + // Assert that the URIs are the same + assert_eq!(first_uri, second_uri, "URIs should be identical"); + + // Assert that the sessions are the same (they should point to the same Arc) + assert!( + Arc::ptr_eq(&first_session, &second_session), + "Sessions should be identical" + ); + shutdown_and_exit(&mut service).await; + }); +} From e59ef827ef903346efe6f603c85b39dc19d650e8 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Tue, 23 Jul 2024 13:02:51 +1000 Subject: [PATCH 16/21] dont clone module cache entry --- sway-core/src/language/ty/module.rs | 2 +- sway-core/src/language/ty/program.rs | 4 +-- sway-core/src/lib.rs | 38 +++++++++++++++++++--- sway-core/src/query_engine/mod.rs | 31 +++++++++++++----- sway-core/src/semantic_analysis/module.rs | 15 +++++---- sway-core/src/semantic_analysis/program.rs | 2 +- 6 files changed, 67 insertions(+), 25 deletions(-) diff --git a/sway-core/src/language/ty/module.rs b/sway-core/src/language/ty/module.rs index 21bc45eb6cf..85234c5825b 100644 --- a/sway-core/src/language/ty/module.rs +++ b/sway-core/src/language/ty/module.rs @@ -23,7 +23,7 @@ pub struct TyModule { #[derive(Clone, Debug)] pub struct TySubmodule { - pub module: TyModule, + pub module: Arc, pub mod_name_span: Span, } diff --git a/sway-core/src/language/ty/program.rs b/sway-core/src/language/ty/program.rs index 7d585533f18..43f85d9d6d3 100644 --- a/sway-core/src/language/ty/program.rs +++ b/sway-core/src/language/ty/program.rs @@ -502,7 +502,7 @@ impl CollectTypesMetadata for TyProgram { for module in std::iter::once(&self.root).chain( self.root .submodules_recursive() - .map(|(_, submod)| &submod.module), + .map(|(_, submod)| &*submod.module), ) { for node in module.all_nodes.iter() { let is_generic_function = node.is_generic_function(decl_engine); @@ -531,7 +531,7 @@ impl CollectTypesMetadata for TyProgram { for module in std::iter::once(&self.root).chain( self.root .submodules_recursive() - .map(|(_, submod)| &submod.module), + .map(|(_, submod)| &*submod.module), ) { for node in module.all_nodes.iter() { if node.is_test_function(decl_engine) { diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 9be80676d8b..9524a2417d8 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -460,9 +460,10 @@ pub(crate) fn is_ty_module_cache_up_to_date( ) -> bool { let query_engine = engines.qe(); let key = ModuleCacheKey::new(path.clone(), include_tests); - let entry = query_engine.get_module_cache_entry(&key); + let cache = query_engine.module_cache.read(); + let entry = cache.get(&key); match entry { - Some(entry) => match entry.typed { + Some(entry) => match &entry.typed { Some(typed) => { let cache_up_to_date = build_config .as_ref() @@ -493,38 +494,61 @@ pub(crate) fn is_parse_module_cache_up_to_date( include_tests: bool, build_config: Option<&BuildConfig>, ) -> bool { + let n1 = std::time::Instant::now(); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + eprintln!("⏱️ split_points took: {:?}", n1.elapsed()); + + let n2 = std::time::Instant::now(); let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); + eprintln!("⏱️ relevant_path took: {:?}", n2.elapsed()); + let n3 = std::time::Instant::now(); let query_engine = engines.qe(); + eprintln!("⏱️ query_engine took: {:?}", n3.elapsed()); + + let n4 = std::time::Instant::now(); let key = ModuleCacheKey::new(path.clone(), include_tests); - let entry = query_engine.get_module_cache_entry(&key); + eprintln!("⏱️ key took: {:?}", n4.elapsed()); + + let n5 = std::time::Instant::now(); + // let entry = query_engine.get_module_cache_entry(&key); + let cache = query_engine.module_cache.read(); + let entry = cache.get(&key); + eprintln!("⏱️ entry took: {:?}", n5.elapsed()); + let res = match entry { Some(entry) => { // Let's check if we can re-use the dependency information // we got from the cache. + let n6 = std::time::Instant::now(); let cache_up_to_date = build_config .as_ref() .and_then(|x| x.lsp_mode.as_ref()) .and_then(|lsp| { + eprintln!("⏱️ cache_up_to_date lsp_mode: {:?}", n6.elapsed()); // First try to get the file version from lsp if it exists lsp.file_versions.get(path.as_ref()) }) .map_or_else( || { + let n7 = std::time::Instant::now(); // Otherwise we can safely read the file from disk here, as the LSP has not modified it, or we are not in LSP mode. // Check if the file has been modified or if its hash is the same as the last compilation let modified_time = std::fs::metadata(path.as_path()) .ok() .and_then(|m| m.modified().ok()); + eprintln!("⏱️ cache_up_to_date modified_time: {:?}", n7.elapsed()); + + let n8 = std::time::Instant::now(); entry.parsed.modified_time == modified_time || { let src = std::fs::read_to_string(path.as_path()).unwrap(); let mut hasher = DefaultHasher::new(); src.hash(&mut hasher); let hash = hasher.finish(); + eprintln!("⏱️ cache_up_to_date hash: {:?}", n8.elapsed()); hash == entry.common.hash } }, @@ -533,16 +557,20 @@ pub(crate) fn is_parse_module_cache_up_to_date( !version.map_or(false, |v| v > entry.parsed.version.unwrap_or(0)) }, ); + eprintln!("⏱️ cache_up_to_date took: {:?}", n6.elapsed()); // Look at the dependencies recursively to make sure they have not been // modified either. if cache_up_to_date { + let n9 = std::time::Instant::now(); //eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); - entry.common.dependencies.iter().all(|path| { + let res = entry.common.dependencies.iter().all(|path| { // eprint!("checking dep path {:?} ", path); is_parse_module_cache_up_to_date(engines, path, include_tests, build_config) - }) + }); + eprintln!("⏱️ is_parse_module_cache_up_to_date: {:?}", n9.elapsed()); + res } else { false } diff --git a/sway-core/src/query_engine/mod.rs b/sway-core/src/query_engine/mod.rs index 79e5aa04bb0..7194f9f5728 100644 --- a/sway-core/src/query_engine/mod.rs +++ b/sway-core/src/query_engine/mod.rs @@ -1,4 +1,4 @@ -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockReadGuard}; use std::{ collections::HashMap, ops::{Deref, DerefMut}, @@ -46,7 +46,7 @@ pub struct ParsedModuleInfo { #[derive(Clone, Debug)] pub struct TypedModuleInfo { - pub module: TyModule, + pub module: Arc, pub modified_time: Option, pub version: Option, } @@ -94,7 +94,7 @@ impl ModuleCacheEntry { } #[derive(Debug, Default, Clone)] -struct ModuleCacheMap(HashMap); +pub struct ModuleCacheMap(HashMap); impl Deref for ModuleCacheMap { type Target = HashMap; @@ -142,16 +142,26 @@ pub struct FunctionCacheEntry { #[derive(Debug, Default, Clone)] pub struct QueryEngine { // We want the below types wrapped in Arcs to optimize cloning from LSP. - module_cache: Arc>, + pub module_cache: Arc>, programs_cache: Arc>, function_cache: Arc>, } impl QueryEngine { - pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option { - let cache = self.module_cache.read(); - cache.get(key).cloned() - } + // pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option<(RwLockReadGuard<'_, ModuleCacheMap>, &ModuleCacheEntry)> { + // let cache = self.module_cache.read(); + // cache.get(key).map(|entry| (cache, entry)) + // } + + // pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option<&ModuleCacheEntry> { + // let cache = self.module_cache.read(); + // cache.get(key) + // } + + // pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option { + // let cache = self.module_cache.read(); + // cache.get(key).cloned() + // } pub fn update_or_insert_parsed_module_cache_entry(&self, entry: ModuleCacheEntry) { let path = entry.common.path.clone(); @@ -167,8 +177,11 @@ impl QueryEngine { } pub fn get_programs_cache_entry(&self, path: &Arc) -> Option { + let now = std::time::Instant::now(); let cache = self.programs_cache.read(); - cache.get(path).cloned() + let res = cache.get(path).cloned(); + eprintln!("⏱️ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!get_programs_cache_entry took: {:?}", now.elapsed()); + res } pub fn insert_programs_cache_entry(&self, entry: ProgramsCacheEntry) { diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index a6d71b519db..c29f1e9266f 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -262,7 +262,7 @@ impl ty::TyModule { source_id: Option<&SourceId>, engines: &Engines, build_config: Option<&BuildConfig>, - ) -> Option { + ) -> Option> { // Check if the module is already in the cache if let Some(source_id) = source_id { let path = engines.se().get_path(&source_id); @@ -275,7 +275,8 @@ impl ty::TyModule { .collect::(); let key: ModuleCacheKey = ModuleCacheKey::new(path.clone().into(), include_tests); - if let Some(entry) = engines.qe().get_module_cache_entry(&key) { + let cache = engines.qe().module_cache.read(); + if let Some(entry) = cache.get(&key) { // We now need to check if the module is up to date, if not, we need to recompute it so // we will return None, otherwise we will return the cached module. @@ -284,7 +285,7 @@ impl ty::TyModule { // Let's check if we can re-use the dependency information // we got from the cache. - if let Some(typed) = entry.typed { + if let Some(typed) = &entry.typed { let is_up_to_date = is_ty_module_cache_up_to_date( engines, &path.into(), @@ -301,7 +302,7 @@ impl ty::TyModule { status, relevant_path, is_up_to_date ); if is_up_to_date { - return Some(typed.module); + return Some(typed.module.clone()); } } } else { @@ -321,7 +322,7 @@ impl ty::TyModule { kind: TreeType, parsed: &ParseModule, build_config: Option<&BuildConfig>, - ) -> Result { + ) -> Result, ErrorEmitted> { let ParseModule { submodules, tree, @@ -458,13 +459,13 @@ impl ty::TyModule { } } - let ty_module = Self { + let ty_module = Arc::new(Self { span: span.clone(), submodules, namespace: ctx.namespace.clone(), all_nodes, attributes: attributes.clone(), - }; + }); // Cache the ty module if let Some(source_id) = span.source_id() { diff --git a/sway-core/src/semantic_analysis/program.rs b/sway-core/src/semantic_analysis/program.rs index 65f5d762552..bfe41511a4f 100644 --- a/sway-core/src/semantic_analysis/program.rs +++ b/sway-core/src/semantic_analysis/program.rs @@ -82,7 +82,7 @@ impl TyProgram { let program = TyProgram { kind, - root, + root: (*root).clone(), declarations, configurables, storage_slots: vec![], From 6e1e4d3d2e0ef01f501aedd9fc0d0987e80f3d66 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Tue, 23 Jul 2024 13:43:29 +1000 Subject: [PATCH 17/21] clean up --- sway-core/src/lib.rs | 36 +++++-------------------------- sway-core/src/query_engine/mod.rs | 20 +---------------- 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 9524a2417d8..a6b85af42b2 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -494,61 +494,38 @@ pub(crate) fn is_parse_module_cache_up_to_date( include_tests: bool, build_config: Option<&BuildConfig>, ) -> bool { - let n1 = std::time::Instant::now(); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; - eprintln!("⏱️ split_points took: {:?}", n1.elapsed()); - - let n2 = std::time::Instant::now(); let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - eprintln!("⏱️ relevant_path took: {:?}", n2.elapsed()); - - let n3 = std::time::Instant::now(); - let query_engine = engines.qe(); - eprintln!("⏱️ query_engine took: {:?}", n3.elapsed()); - - let n4 = std::time::Instant::now(); let key = ModuleCacheKey::new(path.clone(), include_tests); - eprintln!("⏱️ key took: {:?}", n4.elapsed()); - - let n5 = std::time::Instant::now(); - // let entry = query_engine.get_module_cache_entry(&key); - let cache = query_engine.module_cache.read(); + let cache = engines.qe().module_cache.read(); let entry = cache.get(&key); - eprintln!("⏱️ entry took: {:?}", n5.elapsed()); let res = match entry { Some(entry) => { // Let's check if we can re-use the dependency information // we got from the cache. - let n6 = std::time::Instant::now(); let cache_up_to_date = build_config .as_ref() .and_then(|x| x.lsp_mode.as_ref()) .and_then(|lsp| { - eprintln!("⏱️ cache_up_to_date lsp_mode: {:?}", n6.elapsed()); // First try to get the file version from lsp if it exists lsp.file_versions.get(path.as_ref()) }) .map_or_else( || { - let n7 = std::time::Instant::now(); // Otherwise we can safely read the file from disk here, as the LSP has not modified it, or we are not in LSP mode. // Check if the file has been modified or if its hash is the same as the last compilation let modified_time = std::fs::metadata(path.as_path()) .ok() .and_then(|m| m.modified().ok()); - eprintln!("⏱️ cache_up_to_date modified_time: {:?}", n7.elapsed()); - - let n8 = std::time::Instant::now(); entry.parsed.modified_time == modified_time || { let src = std::fs::read_to_string(path.as_path()).unwrap(); let mut hasher = DefaultHasher::new(); src.hash(&mut hasher); let hash = hasher.finish(); - eprintln!("⏱️ cache_up_to_date hash: {:?}", n8.elapsed()); hash == entry.common.hash } }, @@ -557,20 +534,15 @@ pub(crate) fn is_parse_module_cache_up_to_date( !version.map_or(false, |v| v > entry.parsed.version.unwrap_or(0)) }, ); - eprintln!("⏱️ cache_up_to_date took: {:?}", n6.elapsed()); // Look at the dependencies recursively to make sure they have not been // modified either. if cache_up_to_date { - let n9 = std::time::Instant::now(); //eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); - - let res = entry.common.dependencies.iter().all(|path| { + entry.common.dependencies.iter().all(|path| { // eprint!("checking dep path {:?} ", path); is_parse_module_cache_up_to_date(engines, path, include_tests, build_config) - }); - eprintln!("⏱️ is_parse_module_cache_up_to_date: {:?}", n9.elapsed()); - res + }) } else { false } @@ -830,6 +802,7 @@ pub fn compile_to_ast( }; } + let parse_now = std::time::Instant::now(); // Parse the program to a concrete syntax tree (CST). let parse_program_opt = time_expr!( "parse the program to a concrete syntax tree (CST)", @@ -838,6 +811,7 @@ pub fn compile_to_ast( build_config, metrics ); + eprintln!("⏱️ compile_to_ast took {:?}", parse_now.elapsed()); check_should_abort(handler, retrigger_compilation.clone(), 805)?; diff --git a/sway-core/src/query_engine/mod.rs b/sway-core/src/query_engine/mod.rs index 7194f9f5728..d4fd5a6df69 100644 --- a/sway-core/src/query_engine/mod.rs +++ b/sway-core/src/query_engine/mod.rs @@ -148,21 +148,6 @@ pub struct QueryEngine { } impl QueryEngine { - // pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option<(RwLockReadGuard<'_, ModuleCacheMap>, &ModuleCacheEntry)> { - // let cache = self.module_cache.read(); - // cache.get(key).map(|entry| (cache, entry)) - // } - - // pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option<&ModuleCacheEntry> { - // let cache = self.module_cache.read(); - // cache.get(key) - // } - - // pub fn get_module_cache_entry(&self, key: &ModuleCacheKey) -> Option { - // let cache = self.module_cache.read(); - // cache.get(key).cloned() - // } - pub fn update_or_insert_parsed_module_cache_entry(&self, entry: ModuleCacheEntry) { let path = entry.common.path.clone(); let include_tests = entry.common.include_tests; @@ -177,11 +162,8 @@ impl QueryEngine { } pub fn get_programs_cache_entry(&self, path: &Arc) -> Option { - let now = std::time::Instant::now(); let cache = self.programs_cache.read(); - let res = cache.get(path).cloned(); - eprintln!("⏱️ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!get_programs_cache_entry took: {:?}", now.elapsed()); - res + cache.get(path).cloned() } pub fn insert_programs_cache_entry(&self, entry: ProgramsCacheEntry) { From c56d0d263a47ff61d4515c63ff3d1ade9afe909e Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Wed, 24 Jul 2024 12:01:11 +1000 Subject: [PATCH 18/21] update is_ty_module_cache_up_to_date function --- sway-core/src/lib.rs | 49 +++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index a6b85af42b2..c0c0258287c 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -451,41 +451,34 @@ fn parse_module_tree( }) } -// TODO: This function can probably be written better and more concisely. +/// Checks if the typed module cache for a given path is up to date. +/// +/// Returns `true` if the cache is up to date, `false` otherwise. pub(crate) fn is_ty_module_cache_up_to_date( engines: &Engines, path: &Arc, include_tests: bool, build_config: Option<&BuildConfig>, ) -> bool { - let query_engine = engines.qe(); + let cache = engines.qe().module_cache.read(); let key = ModuleCacheKey::new(path.clone(), include_tests); - let cache = query_engine.module_cache.read(); - let entry = cache.get(&key); - match entry { - Some(entry) => match &entry.typed { - Some(typed) => { - let cache_up_to_date = build_config - .as_ref() - .and_then(|x| x.lsp_mode.as_ref()) - .and_then(|lsp| lsp.file_versions.get(path.as_ref())) - .map_or_else( - || false, - |version| !version.map_or(false, |v| v > typed.version.unwrap_or(0)), - ); - if cache_up_to_date { - entry.common.dependencies.iter().all(|path| { - //eprint!("checking dep path {:?} ", path); - is_ty_module_cache_up_to_date(engines, path, include_tests, build_config) - }) - } else { - false - } - } - None => false, - }, - None => false, - } + + cache.get(&key).map_or(false, |entry| { + entry.typed.as_ref().map_or(false, |typed| { + // Check if the cache is up to date based on file versions + let cache_up_to_date = build_config + .and_then(|x| x.lsp_mode.as_ref()) + .and_then(|lsp| lsp.file_versions.get(path.as_ref())) + .map_or(true, |version| { + version.map_or(true, |v| v <= typed.version.unwrap_or(0)) + }); + + // If the cache is up to date, recursively check all dependencies + cache_up_to_date && entry.common.dependencies.iter().all(|dep_path| + is_ty_module_cache_up_to_date(engines, dep_path, include_tests, build_config) + ) + }) + }) } pub(crate) fn is_parse_module_cache_up_to_date( From 3bf0e80d770acf0d2078b1c091ca50557326c55a Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Wed, 24 Jul 2024 12:41:46 +1000 Subject: [PATCH 19/21] clean up and add comments --- sway-core/src/lib.rs | 119 ++++++++---------- sway-core/src/semantic_analysis/module.rs | 146 ++++++++++++---------- 2 files changed, 131 insertions(+), 134 deletions(-) diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index c0c0258287c..01d12f231b0 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -453,7 +453,11 @@ fn parse_module_tree( /// Checks if the typed module cache for a given path is up to date. /// -/// Returns `true` if the cache is up to date, `false` otherwise. +/// This function determines whether the cached typed representation of a module +/// is still valid based on file versions and dependencies. +/// +/// Note: This functionality is currently only supported when the compiler is +/// initiated from the language server. pub(crate) fn is_ty_module_cache_up_to_date( engines: &Engines, path: &Arc, @@ -462,7 +466,6 @@ pub(crate) fn is_ty_module_cache_up_to_date( ) -> bool { let cache = engines.qe().module_cache.read(); let key = ModuleCacheKey::new(path.clone(), include_tests); - cache.get(&key).map_or(false, |entry| { entry.typed.as_ref().map_or(false, |typed| { // Check if the cache is up to date based on file versions @@ -474,13 +477,18 @@ pub(crate) fn is_ty_module_cache_up_to_date( }); // If the cache is up to date, recursively check all dependencies - cache_up_to_date && entry.common.dependencies.iter().all(|dep_path| - is_ty_module_cache_up_to_date(engines, dep_path, include_tests, build_config) - ) + cache_up_to_date + && entry.common.dependencies.iter().all(|dep_path| { + is_ty_module_cache_up_to_date(engines, dep_path, include_tests, build_config) + }) }) }) } +/// Checks if the parsed module cache for a given path is up to date. +/// +/// This function determines whether the cached parsed representation of a module +/// is still valid based on file versions, modification times, or content hashes. pub(crate) fn is_parse_module_cache_up_to_date( engines: &Engines, path: &Arc, @@ -492,69 +500,50 @@ pub(crate) fn is_parse_module_cache_up_to_date( .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - let key = ModuleCacheKey::new(path.clone(), include_tests); + let cache = engines.qe().module_cache.read(); - let entry = cache.get(&key); - - let res = match entry { - Some(entry) => { - // Let's check if we can re-use the dependency information - // we got from the cache. - let cache_up_to_date = build_config - .as_ref() - .and_then(|x| x.lsp_mode.as_ref()) - .and_then(|lsp| { - // First try to get the file version from lsp if it exists - lsp.file_versions.get(path.as_ref()) - }) - .map_or_else( - || { - // Otherwise we can safely read the file from disk here, as the LSP has not modified it, or we are not in LSP mode. - // Check if the file has been modified or if its hash is the same as the last compilation - let modified_time = std::fs::metadata(path.as_path()) - .ok() - .and_then(|m| m.modified().ok()); - entry.parsed.modified_time == modified_time || { - let src = std::fs::read_to_string(path.as_path()).unwrap(); - let mut hasher = DefaultHasher::new(); - src.hash(&mut hasher); - let hash = hasher.finish(); - hash == entry.common.hash - } - }, - |version| { - // The cache is invalid if the lsp version is greater than the last compilation - !version.map_or(false, |v| v > entry.parsed.version.unwrap_or(0)) - }, - ); - - // Look at the dependencies recursively to make sure they have not been - // modified either. - if cache_up_to_date { - //eprintln!("num dependencies for path {:?}: {}", path, entry.dependencies.len()); - entry.common.dependencies.iter().all(|path| { - // eprint!("checking dep path {:?} ", path); - is_parse_module_cache_up_to_date(engines, path, include_tests, build_config) - }) - } else { - false - } - } - None => false, - }; + let key = ModuleCacheKey::new(path.clone(), include_tests); - if res { - eprintln!( - "🀄 👓 Checking cache for parse module {:?} | is up to date? true 🟩", - relevant_path - ); - } else { - eprintln!( - "🀄 👓 Checking cache for parse module {:?} | is up to date? FALSE 🟥", - relevant_path - ); - } + let res = cache.get(&key).map_or(false, |entry| { + // Determine if the cached dependency information is still valid + let cache_up_to_date = build_config + .and_then(|x| x.lsp_mode.as_ref()) + .and_then(|lsp| lsp.file_versions.get(path.as_ref())) + .map_or_else( + || { + // If LSP mode is not active or file version is unavailable, fall back to filesystem checks. + let modified_time = std::fs::metadata(path.as_path()) + .ok() + .and_then(|m| m.modified().ok()); + // Check if modification time matches, or if not, compare file content hash + entry.parsed.modified_time == modified_time || { + let src = std::fs::read_to_string(path.as_path()).unwrap(); + let mut hasher = DefaultHasher::new(); + src.hash(&mut hasher); + hasher.finish() == entry.common.hash + } + }, + |version| { + // In LSP mode, cache is valid if the current version is not greater + // than the version at last compilation. + !version.map_or(false, |v| v > entry.parsed.version.unwrap_or(0)) + }, + ); + + // Checks if the typed module cache for a given path is up to date// If the cache is up to date, recursively check all dependencies to make sure they have not been + // modified either. + cache_up_to_date + && entry.common.dependencies.iter().all(|dep_path| { + is_parse_module_cache_up_to_date(engines, dep_path, include_tests, build_config) + }) + }); + eprintln!( + "🀄 👓 Checking cache for parse module {:?} | is up to date? {} {}", + relevant_path, + if res { "true 🟩" } else { "FALSE 🟥" }, + if res { " " } else { "" } + ); res } diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index c29f1e9266f..002c03bd574 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -11,12 +11,12 @@ use sway_error::{ error::CompileError, handler::{ErrorEmitted, Handler}, }; -use sway_types::{BaseIdent, Named, SourceId, Spanned}; +use sway_types::{BaseIdent, Named, SourceId}; use crate::{ decl_engine::{DeclEngineGet, DeclId}, engine_threading::DebugWithEngines, - is_parse_module_cache_up_to_date, is_ty_module_cache_up_to_date, + is_ty_module_cache_up_to_date, language::{ parsed::*, ty::{self, TyAstNodeContent, TyDecl}, @@ -258,58 +258,60 @@ impl ty::TyModule { Ok(()) } - fn check_cache( + /// Retrieves a cached typed module if it's up to date. + /// + /// This function checks the cache for a typed module corresponding to the given source ID. + /// If found and up to date, it returns the cached module. Otherwise, it returns None. + fn get_cached_ty_module_if_up_to_date( source_id: Option<&SourceId>, engines: &Engines, build_config: Option<&BuildConfig>, ) -> Option> { - // Check if the module is already in the cache - if let Some(source_id) = source_id { - let path = engines.se().get_path(&source_id); - let include_tests = build_config.map_or(false, |x| x.include_tests); + let Some(source_id) = source_id else { + return None; + }; - let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; - let relevant_path = path - .iter() - .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) - .collect::(); + // Create a cache key and get the module cache + let path = engines.se().get_path(&source_id); + let include_tests = build_config.map_or(false, |x| x.include_tests); + let key = ModuleCacheKey::new(path.clone().into(), include_tests); + let cache = engines.qe().module_cache.read(); - let key: ModuleCacheKey = ModuleCacheKey::new(path.clone().into(), include_tests); - let cache = engines.qe().module_cache.read(); - if let Some(entry) = cache.get(&key) { - // We now need to check if the module is up to date, if not, we need to recompute it so - // we will return None, otherwise we will return the cached module. + let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let relevant_path = path + .iter() + .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) + .collect::(); - // For now we will duplicate the logic of the parsed cache entry check but - // we should ideally have a way of sharing this logic. + cache.get(&key).and_then(|entry| { + entry.typed.as_ref().and_then(|typed| { + // Check if the cached module is up to date + let is_up_to_date = is_ty_module_cache_up_to_date( + engines, + &path.into(), + include_tests, + build_config, + ); - // Let's check if we can re-use the dependency information - // we got from the cache. - if let Some(typed) = &entry.typed { - let is_up_to_date = is_ty_module_cache_up_to_date( - engines, - &path.into(), - include_tests, - build_config, - ); - let status = if is_up_to_date { - "✅ Cache hit" - } else { - "🔄 Cache miss" - }; - eprintln!( - "{} for module {:?} (up to date: {})", - status, relevant_path, is_up_to_date - ); - if is_up_to_date { - return Some(typed.module.clone()); - } + // Log the cache status + let status = if is_up_to_date { + "✅ Cache hit" + } else { + "🔄 Cache miss" + }; + eprintln!( + "{} for module {:?} (up to date: {})", + status, relevant_path, is_up_to_date + ); + + // Return the cached module if it's up to date, otherwise None + if is_up_to_date { + Some(typed.module.clone()) + } else { + None } - } else { - eprintln!("❌ No cache entry for module {:?}", relevant_path); - } - } - None + }) + }) } /// Type-check the given parsed module to produce a typed module. @@ -337,48 +339,54 @@ impl ty::TyModule { "Root Module: {:?}", parsed.span.source_id().map(|x| engines.se().get_path(x)) ); - if let Some(module) = - ty::TyModule::check_cache(parsed.span.source_id(), engines, build_config) - { + + // Try to get the cached root module + if let Some(module) = ty::TyModule::get_cached_ty_module_if_up_to_date( + parsed.span.source_id(), + engines, + build_config, + ) { return Ok(module); } // Type-check submodules first in order of evaluation previously computed by the dependency graph. - let submodules_res = module_eval_order + let submodules_res: Result, _> = module_eval_order .iter() .map(|eval_mod_name| { let (name, submodule) = submodules .iter() - .find(|(submod_name, _submodule)| eval_mod_name == submod_name) + .find(|(submod_name, _)| eval_mod_name == submod_name) .unwrap(); - // Check if the submodule cache is up to date - if let Some(module) = ty::TyModule::check_cache( + // Try to get the cached submodule + if let Some(cached_module) = ty::TyModule::get_cached_ty_module_if_up_to_date( submodule.module.span.source_id(), engines, build_config, ) { - let submodule = ty::TySubmodule { - module, - mod_name_span: submodule.mod_name_span.clone(), - }; - Ok((name.clone(), submodule)) - } else { - Ok(( + // If cached, create TySubmodule from cached module + Ok::<(BaseIdent, ty::TySubmodule), ErrorEmitted>(( name.clone(), - ty::TySubmodule::type_check( - handler, - ctx.by_ref(), - engines, - name.clone(), - kind, - submodule, - build_config, - )?, + ty::TySubmodule { + module: cached_module, + mod_name_span: submodule.mod_name_span.clone(), + }, )) + } else { + // If not cached, type-check the submodule + let type_checked_submodule = ty::TySubmodule::type_check( + handler, + ctx.by_ref(), + engines, + name.clone(), + kind, + submodule, + build_config, + )?; + Ok((name.clone(), type_checked_submodule)) } }) - .collect::, _>>(); + .collect(); // TODO: Ordering should be solved across all modules prior to the beginning of type-check. let ordered_nodes = node_dependencies::order_ast_nodes_by_dependency( From c14a8f4e77f07ae38a083aa2e3bfca5d713568fe Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Wed, 24 Jul 2024 12:51:06 +1000 Subject: [PATCH 20/21] clippy --- sway-core/Cargo.toml | 1 - sway-core/src/lib.rs | 3 --- sway-core/src/query_engine/mod.rs | 3 +-- sway-core/src/semantic_analysis/module.rs | 12 +++--------- sway-lsp/src/core/session.rs | 1 - 5 files changed, 4 insertions(+), 16 deletions(-) diff --git a/sway-core/Cargo.toml b/sway-core/Cargo.toml index a903dca0b7b..9c0122123c2 100644 --- a/sway-core/Cargo.toml +++ b/sway-core/Cargo.toml @@ -34,7 +34,6 @@ parking_lot = "0.12" pest = "2.1.3" pest_derive = "2.1" petgraph = "0.6" -rayon = "1.5" rustc-hash = "1.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.91" diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 01d12f231b0..52e261b028a 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -31,12 +31,9 @@ pub use asm_generation::{CompiledBytecode, FinalizedEntry}; pub use build_config::{BuildConfig, BuildTarget, LspConfig, OptLevel, PrintAsm, PrintIr}; use control_flow_analysis::ControlFlowGraph; pub use debug_generation::write_dwarf; -use decl_engine::parsed_engine; -use fuel_vm::fuel_merkle::common; use indexmap::IndexMap; use metadata::MetadataManager; use query_engine::{ModuleCacheKey, ModuleCommonInfo, ParsedModuleInfo, ProgramsCacheEntry}; -use rayon::iter::{ParallelBridge, ParallelIterator}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; diff --git a/sway-core/src/query_engine/mod.rs b/sway-core/src/query_engine/mod.rs index d4fd5a6df69..b25a64dffef 100644 --- a/sway-core/src/query_engine/mod.rs +++ b/sway-core/src/query_engine/mod.rs @@ -1,4 +1,4 @@ -use parking_lot::{RwLock, RwLockReadGuard}; +use parking_lot::RwLock; use std::{ collections::HashMap, ops::{Deref, DerefMut}, @@ -47,7 +47,6 @@ pub struct ParsedModuleInfo { #[derive(Clone, Debug)] pub struct TypedModuleInfo { pub module: Arc, - pub modified_time: Option, pub version: Option, } diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 002c03bd574..3874c234331 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -267,12 +267,10 @@ impl ty::TyModule { engines: &Engines, build_config: Option<&BuildConfig>, ) -> Option> { - let Some(source_id) = source_id else { - return None; - }; + let source_id = source_id?; // Create a cache key and get the module cache - let path = engines.se().get_path(&source_id); + let path = engines.se().get_path(source_id); let include_tests = build_config.map_or(false, |x| x.include_tests); let key = ModuleCacheKey::new(path.clone().into(), include_tests); let cache = engines.qe().module_cache.read(); @@ -477,16 +475,13 @@ impl ty::TyModule { // Cache the ty module if let Some(source_id) = span.source_id() { - let path = engines.se().get_path(&source_id); + let path = engines.se().get_path(source_id); let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) .collect::(); - let modified_time = std::fs::metadata(path.as_path()) - .ok() - .and_then(|m| m.modified().ok()); let version = build_config .and_then(|config| config.lsp_mode.as_ref()) .and_then(|lsp| lsp.file_versions.get(&path).copied()) @@ -499,7 +494,6 @@ impl ty::TyModule { &key, TypedModuleInfo { module: ty_module.clone(), - modified_time, version, }, ); diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index a25449bc43a..25963211948 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -43,7 +43,6 @@ use sway_core::{ use sway_error::{error::CompileError, handler::Handler, warning::CompileWarning}; use sway_types::{ProgramId, SourceEngine, Spanned}; use sway_utils::{helpers::get_sway_files, PerformanceData}; -use swayfmt::parse; pub type RunnableMap = DashMap>>; pub type ProjectDirectory = PathBuf; From 052ff7f1bc05f6c6b558026b2ba1d3fbac2380d1 Mon Sep 17 00:00:00 2001 From: JoshuaBatty Date: Thu, 25 Jul 2024 12:09:56 +1000 Subject: [PATCH 21/21] lots and lots of dbg prints wip prob safe to remove --- forc-pkg/src/manifest/mod.rs | 22 ++++++++ forc-pkg/src/pkg.rs | 10 +++- sway-core/src/lib.rs | 42 ++++++++++++--- sway-core/src/semantic_analysis/module.rs | 4 +- sway-error/src/handler.rs | 1 + sway-lsp/src/capabilities/diagnostic.rs | 4 +- sway-lsp/src/core/session.rs | 16 +++++- sway-lsp/src/handlers/notification.rs | 2 + sway-lsp/tests/integration/lsp.rs | 1 + sway-lsp/tests/lib.rs | 62 ++++++++++++++++++++++- sway-lsp/tests/utils/src/lib.rs | 2 +- sway-parse/src/lib.rs | 2 + sway-parse/src/module.rs | 10 +++- sway-parse/src/parser.rs | 5 ++ 14 files changed, 165 insertions(+), 18 deletions(-) diff --git a/forc-pkg/src/manifest/mod.rs b/forc-pkg/src/manifest/mod.rs index ca35a46e9da..87c31b6903e 100644 --- a/forc-pkg/src/manifest/mod.rs +++ b/forc-pkg/src/manifest/mod.rs @@ -357,11 +357,33 @@ impl PackageManifestFile { /// Produces the string of the entry point file. pub fn entry_string(&self) -> Result> { + dbg!(); let entry_path = self.entry_path(); + dbg!(); let entry_string = std::fs::read_to_string(entry_path)?; + dbg!(); + if entry_string.is_empty() { + eprintln!("entry_string empty! | entry_path: {:?}", self.entry_path()); + } else { + eprintln!("entry_string not empty! | entry_path: {:?}", self.entry_path()); + } Ok(Arc::from(entry_string)) } + // we could try to read the file 3 times, but it feels like a hack + // pub fn entry_string(&self) -> Result> { + // for attempt in 1..=3 { + // let entry_path = self.entry_path(); + // let entry_string = std::fs::read_to_string(&entry_path)?; + // if !entry_string.is_empty() { + // return Ok(Arc::from(entry_string)); + // } + // eprintln!("Attempt {}: File empty, retrying...", attempt); + // std::thread::sleep(std::time::Duration::from_millis(100)); + // } + // Err(anyhow::anyhow!("File remained empty after multiple attempts")) + // } + /// Parse and return the associated project's program type. pub fn program_type(&self) -> Result { let entry_string = self.entry_string()?; diff --git a/forc-pkg/src/pkg.rs b/forc-pkg/src/pkg.rs index cbd38f05359..91ef1715dde 100644 --- a/forc-pkg/src/pkg.rs +++ b/forc-pkg/src/pkg.rs @@ -2700,7 +2700,9 @@ pub fn check( .with_lsp_mode(lsp_mode.clone()); eprintln!("⏱️ Build config took {:?}", build_config_now.elapsed()); + eprintln!("Forc pacakge loading input string"); let input = manifest.entry_string()?; + eprintln!("Forc package input string loaded | {}", input.clone()); let handler = Handler::default(); let compile_to_ast_now = std::time::Instant::now(); let programs_res = sway_core::compile_to_ast( @@ -2712,6 +2714,7 @@ pub fn check( &pkg.name, retrigger_compilation.clone(), ); + eprintln!("programs res: {:?}", programs_res.is_ok()); eprintln!("⏱️ Compile to AST took {:?}", compile_to_ast_now.elapsed()); if retrigger_compilation @@ -2723,8 +2726,13 @@ pub fn check( } let programs = match programs_res.as_ref() { - Ok(programs) => programs, + Ok(programs) => { + dbg!(); + programs + }, _ => { + eprintln!("ERROR PARSING MODULE | {:?}", programs_res.clone().ok()); + eprintln!("Returning results with handler | {:?}", handler); results.push((programs_res.ok(), handler)); return Ok(results); } diff --git a/sway-core/src/lib.rs b/sway-core/src/lib.rs index 52e261b028a..0653a30776b 100644 --- a/sway-core/src/lib.rs +++ b/sway-core/src/lib.rs @@ -90,6 +90,7 @@ pub fn parse( engines: &Engines, config: Option<&BuildConfig>, ) -> Result<(lexed::LexedProgram, parsed::ParseProgram), ErrorEmitted> { + dbg!(); match config { None => parse_in_memory( handler, @@ -257,13 +258,15 @@ fn parse_submodules( experimental: ExperimentalFlags, lsp_mode: Option<&LspConfig>, ) -> Submodules { + dbg!(); // Assume the happy path, so there'll be as many submodules as dependencies, but no more. let mut submods = Vec::with_capacity(module.submodules().count()); - + dbg!(); module.submodules().for_each(|submod| { // Read the source code from the dependency. // If we cannot, record as an error, but continue with other files. let submod_path = Arc::new(module_path(module_dir, module_name, submod)); + dbg!(); let submod_str: Arc = match std::fs::read_to_string(&*submod_path) { Ok(s) => Arc::from(s), Err(e) => { @@ -275,7 +278,7 @@ fn parse_submodules( return; } }; - + dbg!(); if let Ok(ParsedModuleTree { tree_type: kind, lexed_module, @@ -318,7 +321,7 @@ fn parse_submodules( submods.push(submodule); } }); - + dbg!(); submods } @@ -345,14 +348,23 @@ fn parse_module_tree( experimental: ExperimentalFlags, lsp_mode: Option<&LspConfig>, ) -> Result { + dbg!(); let query_engine = engines.qe(); let lexed_now = std::time::Instant::now(); // Parse this module first. let module_dir = path.parent().expect("module file has no parent directory"); let source_id = engines.se().get_source_id(&path.clone()); - let module = sway_parse::parse_file(handler, src.clone(), Some(source_id))?; - + dbg!(); + let module = match sway_parse::parse_file(handler, src.clone(), Some(source_id)) { + Ok(module) => module, + Err(e) => { + eprintln!("ERROR PARSING MODULE | {:?} | src_file: {}", e, src.clone()); + return Err(e); + } + }; + //let module = sway_parse::parse_file(handler, src.clone(), Some(source_id))?; + dbg!(); // Parse all submodules before converting to the `ParseTree`. // This always recovers on parse errors for the file itself by skipping that file. let submodules = parse_submodules( @@ -366,6 +378,7 @@ fn parse_module_tree( experimental, lsp_mode, ); + dbg!(); eprintln!("⏱️ Lexed module took {:?}", lexed_now.elapsed()); let parsed_now = std::time::Instant::now(); @@ -376,8 +389,11 @@ fn parse_module_tree( engines, module.value.clone(), )?; + dbg!(); let module_kind_span = module.value.kind.span(); + dbg!(); let attributes = module_attrs_to_map(handler, &module.attribute_list)?; + dbg!(); eprintln!("⏱️ Parsed module took {:?}", parsed_now.elapsed()); let lexed_submodules = submodules @@ -429,7 +445,7 @@ fn parse_module_tree( }; let cache_entry = ModuleCacheEntry::new(common_info, parsed_info); - let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let split_points = ["sway-lib-core", "sway-lib-std", "libraries", "multi-trove-getter-contract"]; let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) @@ -492,7 +508,7 @@ pub(crate) fn is_parse_module_cache_up_to_date( include_tests: bool, build_config: Option<&BuildConfig>, ) -> bool { - let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let split_points = ["sway-lib-core", "sway-lib-std", "libraries", "multi-trove-getter-contract"]; let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) @@ -777,10 +793,14 @@ pub fn compile_to_ast( let (warnings, errors) = entry.handler_data; let new_handler = Handler::from_parts(warnings, errors); handler.append(new_handler); + eprintln!("programs cache valid, returning"); return Ok(entry.programs); }; } + eprintln!("programs cache invalid, continuing to parse"); + let input_clone = input.clone(); + let parse_now = std::time::Instant::now(); // Parse the program to a concrete syntax tree (CST). let parse_program_opt = time_expr!( @@ -795,8 +815,14 @@ pub fn compile_to_ast( check_should_abort(handler, retrigger_compilation.clone(), 805)?; let (lexed_program, mut parsed_program) = match parse_program_opt { - Ok(modules) => modules, + Ok(modules) => { + dbg!(); + modules + }, Err(e) => { + dbg!(); + // Input string is completely empty. how? + eprintln!("ERROR PARSING PROGRAM | {:?} | src_file: {}", e, input_clone); handler.dedup(); return Err(e); } diff --git a/sway-core/src/semantic_analysis/module.rs b/sway-core/src/semantic_analysis/module.rs index 3874c234331..5feacaa7f76 100644 --- a/sway-core/src/semantic_analysis/module.rs +++ b/sway-core/src/semantic_analysis/module.rs @@ -275,7 +275,7 @@ impl ty::TyModule { let key = ModuleCacheKey::new(path.clone().into(), include_tests); let cache = engines.qe().module_cache.read(); - let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let split_points = ["sway-lib-core", "sway-lib-std", "libraries", "multi-trove-getter-contract"]; let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) @@ -476,7 +476,7 @@ impl ty::TyModule { // Cache the ty module if let Some(source_id) = span.source_id() { let path = engines.se().get_path(source_id); - let split_points = ["sway-lib-core", "sway-lib-std", "libraries"]; + let split_points = ["sway-lib-core", "sway-lib-std", "libraries", "multi-trove-getter-contract"]; let relevant_path = path .iter() .skip_while(|&comp| !split_points.contains(&comp.to_str().unwrap())) diff --git a/sway-error/src/handler.rs b/sway-error/src/handler.rs index 24d2222da1b..dc933fbe91c 100644 --- a/sway-error/src/handler.rs +++ b/sway-error/src/handler.rs @@ -35,6 +35,7 @@ impl Handler { // Compilation should be cancelled. pub fn cancel(&self) -> ErrorEmitted { + eprintln!("CANCELLED"); ErrorEmitted { _priv: () } } diff --git a/sway-lsp/src/capabilities/diagnostic.rs b/sway-lsp/src/capabilities/diagnostic.rs index d5121d21914..71a860b7a71 100644 --- a/sway-lsp/src/capabilities/diagnostic.rs +++ b/sway-lsp/src/capabilities/diagnostic.rs @@ -42,6 +42,7 @@ pub fn get_diagnostics( errors: &[CompileError], source_engine: &SourceEngine, ) -> DiagnosticMap { + dbg!(); let mut diagnostics = DiagnosticMap::new(); for warning in warnings { let diagnostic = get_warning_diagnostic(warning); @@ -54,6 +55,7 @@ pub fn get_diagnostics( .push(diagnostic); } } + dbg!(); for error in errors { let diagnostic = get_error_diagnostic(error); if let Some(source_id) = error.span().source_id() { @@ -61,7 +63,7 @@ pub fn get_diagnostics( diagnostics.entry(path).or_default().errors.push(diagnostic); } } - + dbg!(); diagnostics } diff --git a/sway-lsp/src/core/session.rs b/sway-lsp/src/core/session.rs index 25963211948..1b09e7ce165 100644 --- a/sway-lsp/src/core/session.rs +++ b/sway-lsp/src/core/session.rs @@ -314,6 +314,8 @@ pub fn traverse( diagnostics = current_diagnostics; if value.is_none() { + eprintln!("Unable to traverse module, value is None"); + // Should this be an error? continue; } let Programs { @@ -428,17 +430,24 @@ pub fn parse_project( let diagnostics = traverse(results, engines, session.clone())?; eprintln!("⏱️ Traversing the ASTS took {:?}", traverse_now.elapsed()); if let Some(config) = &lsp_mode { + dbg!(); // Only write the diagnostics results on didSave or didOpen. if !config.optimized_build { + dbg!(); if let Some((errors, warnings)) = &diagnostics { + dbg!(); *session.diagnostics.write() = capabilities::diagnostic::get_diagnostics(warnings, errors, engines.se()); } } } + dbg!(); let runnables_now = std::time::Instant::now(); if let Some(typed) = &session.compiled_program.read().typed { + dbg!(); session.runnables.clear(); + dbg!(); + // This is where it's crashing OOB into the concurrent slab create_runnables(&session.runnables, typed, engines.de(), engines.se()); } eprintln!("⏱️ creating runnables took: {:?}", runnables_now.elapsed()); @@ -499,8 +508,10 @@ fn create_runnables( decl_engine: &DeclEngine, source_engine: &SourceEngine, ) { + dbg!(); let _p = tracing::trace_span!("create_runnables").entered(); // Insert runnable test functions. + for (decl, _) in typed_program.test_fns(decl_engine) { // Get the span of the first attribute if it exists, otherwise use the span of the function name. let span = decl @@ -508,6 +519,7 @@ fn create_runnables( .first() .map_or_else(|| decl.name.span(), |(_, attr)| attr.span.clone()); if let Some(source_id) = span.source_id() { + dbg!(); let path = source_engine.get_path(source_id); let runnable = Box::new(RunnableTestFn { range: token::get_range_from_span(&span.clone()), @@ -517,15 +529,17 @@ fn create_runnables( runnables.entry(path).or_default().push(runnable); } } - + dbg!(); // Insert runnable main function if the program is a script. if let ty::TyProgramKind::Script { entry_function: ref main_function, .. } = typed_program.kind { + dbg!(); let main_function = decl_engine.get_function(main_function); let span = main_function.name.span(); + dbg!(); if let Some(source_id) = span.source_id() { let path = source_engine.get_path(source_id); let runnable = Box::new(RunnableMainFn { diff --git a/sway-lsp/src/handlers/notification.rs b/sway-lsp/src/handlers/notification.rs index 68c35a99802..3ae1f757fbf 100644 --- a/sway-lsp/src/handlers/notification.rs +++ b/sway-lsp/src/handlers/notification.rs @@ -89,6 +89,8 @@ pub async fn handle_did_change_text_document( state: &ServerState, params: DidChangeTextDocumentParams, ) -> Result<(), LanguageServerError> { + // TODO: Can we cache paths with files that are dirty so we don't need + // to do filesystem IO here for each change? if let Err(err) = document::mark_file_as_dirty(¶ms.text_document.uri) { tracing::warn!("Failed to mark file as dirty: {}", err); } diff --git a/sway-lsp/tests/integration/lsp.rs b/sway-lsp/tests/integration/lsp.rs index e0d063871be..576477081f9 100644 --- a/sway-lsp/tests/integration/lsp.rs +++ b/sway-lsp/tests/integration/lsp.rs @@ -43,6 +43,7 @@ pub(crate) async fn initialize_request(service: &mut LspService) -> 1.into(), json!({ "capabilities": sway_lsp::server_capabilities() }), ); + eprintln!("initialize response: {:?}", response); assert_json_eq!(expected, response.ok().unwrap()); initialize } diff --git a/sway-lsp/tests/lib.rs b/sway-lsp/tests/lib.rs index acdb22ffe3b..b9527520428 100644 --- a/sway-lsp/tests/lib.rs +++ b/sway-lsp/tests/lib.rs @@ -242,12 +242,21 @@ fn did_change_stress_test_random_wait() { // .join("generics_in_contract"); // let uri = init_and_open(&mut service, example_dir.join("src/main.sw")).await; + // let uri = init_and_open( + // &mut service, + // PathBuf::from( + // "/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries", + // ) + // .join("src/fpt_staking_interface.sw"), + // ) + // .await; + let uri = init_and_open( &mut service, PathBuf::from( - "/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/libraries", + "/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/contracts/multi-trove-getter-contract", ) - .join("src/fpt_staking_interface.sw"), + .join("src/main.sw"), ) .await; @@ -290,6 +299,55 @@ fn did_change_stress_test_random_wait() { }); } + +#[test] +fn did_change_stress_test_enter_uzi() { + run_async!({ + let test_duration = tokio::time::Duration::from_secs(250 * 60); // 5 minutes timeout + let test_future = async { + setup_panic_hook(); + let (mut service, _) = LspService::new(ServerState::new); + let uri = init_and_open( + &mut service, + PathBuf::from( + "/Users/josh/Documents/rust/fuel/user_projects/fluid-protocol/contracts/multi-trove-getter-contract", + ) + .join("src/main.sw"), + ) + .await; + + let times = 6000; + for version in 0..times { + //eprintln!("version: {}", version); + let _ = lsp::did_change_request(&mut service, &uri, version + 1, None).await; + if version == 0 { + service.inner().wait_for_parsing().await; + } + // wait for a random amount of time between 1s + tokio::time::sleep(tokio::time::Duration::from_millis(2)).await; + + // there is a 10% chance that a longer 100-800ms wait will be added + if rand::random::() % 100 < 2 { + tokio::time::sleep(tokio::time::Duration::from_millis( + rand::random::() % 100, + )) + .await; + } + } + shutdown_and_exit(&mut service).await; + }; + if tokio::time::timeout(test_duration, test_future) + .await + .is_err() + { + panic!( + "did_change_stress_test_random_wait did not complete within the timeout period." + ); + } + }); +} + + fn garbage_collection_runner(path: PathBuf) { run_async!({ setup_panic_hook(); diff --git a/sway-lsp/tests/utils/src/lib.rs b/sway-lsp/tests/utils/src/lib.rs index 3124481ca06..e60363e6aa4 100644 --- a/sway-lsp/tests/utils/src/lib.rs +++ b/sway-lsp/tests/utils/src/lib.rs @@ -142,7 +142,7 @@ pub async fn random_delay() { /// Sets up the environment and a custom panic hook to print panic information and exit the program. pub fn setup_panic_hook() { // Enable backtrace to get more information about panic - std::env::set_var("RUST_BACKTRACE", "1"); + std::env::set_var("RUST_BACKTRACE", "FULL"); // Take the default panic hook let default_panic = std::panic::take_hook(); diff --git a/sway-parse/src/lib.rs b/sway-parse/src/lib.rs index 2022c954d7a..191e1c2c0ae 100644 --- a/sway-parse/src/lib.rs +++ b/sway-parse/src/lib.rs @@ -42,7 +42,9 @@ pub fn parse_file( src: Arc, source_id: Option, ) -> Result, ErrorEmitted> { + dbg!(); let ts = lex(handler, &src, 0, src.len(), source_id)?; + dbg!(); let (m, _) = Parser::new(handler, &ts).parse_to_end()?; Ok(m) } diff --git a/sway-parse/src/module.rs b/sway-parse/src/module.rs index edd33c8bb9c..047ad4c4af0 100644 --- a/sway-parse/src/module.rs +++ b/sway-parse/src/module.rs @@ -21,6 +21,7 @@ impl Parse for ModuleKind { } else if let Some(library_token) = parser.take() { Ok(Self::Library { library_token }) } else { + eprintln!("❔⁉️ Expected Module Kind, how the hell did we get here?"); Err(parser.emit_error(ParseErrorKind::ExpectedModuleKind)) } } @@ -28,10 +29,13 @@ impl Parse for ModuleKind { impl ParseToEnd for Annotated { fn parse_to_end<'a, 'e>(mut parser: Parser<'a, '_>) -> ParseResult<(Self, ParserConsumed<'a>)> { + dbg!(); // Parse the attribute list. let mut attribute_list = Vec::new(); while let Some(DocComment { .. }) = parser.peek() { + dbg!(); let doc_comment = parser.parse::()?; + dbg!(); // TODO: Use a Literal instead of an Ident when Attribute args // start supporting them and remove `Ident::new_no_trim`. let name = Ident::new_no_trim(doc_comment.content_span.clone()); @@ -55,14 +59,16 @@ impl ParseToEnd for Annotated { ), }), DocStyle::Outer => { + dbg!(); parser.emit_error(ParseErrorKind::ExpectedModuleDocComment); } } } + dbg!(); let (kind, semicolon_token) = parser.parse()?; - + dbg!(); let (items, consumed) = parser.parse_to_end()?; - + dbg!(); let module = Annotated { attribute_list, value: Module { diff --git a/sway-parse/src/parser.rs b/sway-parse/src/parser.rs index ad9d3e7ffc2..3aa77a1e465 100644 --- a/sway-parse/src/parser.rs +++ b/sway-parse/src/parser.rs @@ -209,6 +209,8 @@ impl<'a, 'e> Parser<'a, 'e> { /// Parses a `T` in its canonical way. pub fn parse(&mut self) -> ParseResult { + //eprintln!("parse Type of T: {}", std::any::type_name::()); + T::parse(self) } @@ -220,6 +222,9 @@ impl<'a, 'e> Parser<'a, 'e> { } pub fn parse_to_end(self) -> ParseResult<(T, ParserConsumed<'a>)> { + // eprintln! type of T + // eprintln!("parse_to_end Type of T: {}", std::any::type_name::()); + T::parse_to_end(self) }