From 648568ff839e1d8d1bfc72d800d3dd4b1d6cdbf3 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 6 Mar 2020 17:52:33 -0800 Subject: [PATCH 01/11] WIP new module importer --- Cargo.toml | 3 ++ core/lib.rs | 2 + core/module_importer.rs | 112 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 core/module_importer.rs diff --git a/Cargo.toml b/Cargo.toml index b273e4222f2448..208edf83da2d03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,6 @@ members = [ "deno_typescript", "test_plugin" ] + +[replace] +"rusty_v8:0.3.4" = { path = 'd:/v8rs' } diff --git a/core/lib.rs b/core/lib.rs index c4fbb33f4eec6f..70f4ca49449e6f 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -14,6 +14,7 @@ mod es_isolate; mod flags; mod isolate; mod js_errors; +mod module_importer; mod module_specifier; mod modules; mod ops; @@ -28,6 +29,7 @@ pub use crate::es_isolate::*; pub use crate::flags::v8_set_flags; pub use crate::isolate::*; pub use crate::js_errors::*; +pub use crate::module_importer::*; pub use crate::module_specifier::*; pub use crate::modules::*; pub use crate::ops::*; diff --git a/core/module_importer.rs b/core/module_importer.rs new file mode 100644 index 00000000000000..a5940755c83ef3 --- /dev/null +++ b/core/module_importer.rs @@ -0,0 +1,112 @@ +use std::borrow::Borrow; +use std::collections::HashMap; +use std::collections::HashSet; +use std::future::Future; +use std::hash::Hash; +use std::hash::Hasher; +use std::pin::Pin; +use std::rc::Rc; + +use futures::future::Shared; +use futures::future::TryFuture; +use futures::FutureExt; +use futures::TryFutureExt; +use rusty_v8 as v8; +use url::Url; + +use crate::modules::ModuleSource as LegacyModuleSource; +use crate::ErrBox; +use crate::ModuleLoader as ModuleFetcher; +use crate::ModuleSpecifier; + +pub struct Module { + // URL after resolving the import specifier to an absolute URL, but without + // taking into any HTTP redirects. + resolved_url: Url, + source: + Shared>>>>>, +} + +impl Borrow for Module { + fn borrow(&self) -> &Url { + &self.resolved_url + } +} + +impl Eq for Module {} + +impl PartialEq for Module { + fn eq(&self, other: &Self) -> bool { + self.resolved_url == other.resolved_url + } +} + +impl Hash for Module { + fn hash(&self, state: &mut H) { + self.resolved_url.hash(state) + } +} + +pub struct ModuleSource { + // URL that the source was actually loaded from, according to the loader. It + // might differ from the specified URL for various reasons, e.g. because + // an HTTP redirect happened while fetching from the specified URL.Future + // If this module's source code contains import statements that specify + // relative URLs, the fetched URL will be used as the basis for their + // resolution. + fetched_url: Url, + // JavaScript source code that is compiled by V8. + js_source_code: String, +} + +impl From for ModuleSource { + fn from(v: LegacyModuleSource) -> Self { + Self { + fetched_url: Url::parse(&v.module_url_found).unwrap(), + js_source_code: v.code, + } + } +} + +pub struct ModuleImporter { + fetcher: Rc, + modules: HashSet, +} + +impl ModuleImporter { + pub fn new(fetcher: Rc) -> Self { + Self { + fetcher, + modules: HashSet::new(), + } + } + + // The `is_dyn_import` flag is forwarded to the module loader if the module's + // source hasn't been loaded yet. If the module already exists in the module + // set, it is ignored. + pub fn get(&mut self, resolved_url: &Url, is_dyn_import: bool) -> &Module { + // Note: future versions of Rust will likely implement a method like + // `get_or_insert_with()` or `get_or_insert_owned()` on HashSet. It would + // be more efficient to use these when they become available. + if !self.modules.contains(resolved_url) { + let fetcher = self.fetcher.clone(); + let module_specifier = resolved_url.clone().into(); + let module = Module { + resolved_url: resolved_url.clone(), + source: async move { + Rc::new( + fetcher + .load(&module_specifier, None, is_dyn_import) + .map_ok(ModuleSource::from) + .await, + ) + } + .boxed_local() + .shared(), + }; + self.modules.insert(module); + } + + self.modules.get(resolved_url).unwrap() + } +} From 79d96e91a664cc8210eb3a58da89f3863e343531 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Mon, 9 Mar 2020 17:28:45 -0700 Subject: [PATCH 02/11] wip --- cli/state.rs | 1 + core/lib.rs | 2 +- core/module_importer.rs | 119 ++++++++++++++++++++++++++++++++-------- core/modules.rs | 4 ++ 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/cli/state.rs b/cli/state.rs index 44ab73034ec6b2..94dc4f5717cec7 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -9,6 +9,7 @@ use crate::ops::JsonOp; use crate::ops::MinimalOp; use crate::permissions::DenoPermissions; use crate::worker::WorkerHandle; +use deno_core::v8; use deno_core::Buf; use deno_core::CoreOp; use deno_core::ErrBox; diff --git a/core/lib.rs b/core/lib.rs index 70f4ca49449e6f..e60cf55837738e 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -22,7 +22,7 @@ mod plugins; mod resources; mod shared_queue; -use rusty_v8 as v8; +pub use rusty_v8 as v8; pub use crate::any_error::*; pub use crate::es_isolate::*; diff --git a/core/module_importer.rs b/core/module_importer.rs index a5940755c83ef3..56cf453c01abf1 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -14,19 +14,32 @@ use futures::TryFutureExt; use rusty_v8 as v8; use url::Url; +use crate::bindings::module_origin; use crate::modules::ModuleSource as LegacyModuleSource; use crate::ErrBox; -use crate::ModuleLoader as ModuleFetcher; +use crate::ModuleLoader; use crate::ModuleSpecifier; +pub enum ImportError { + Rust(ErrBox), + JS(v8::Global), +} + +pub type ImportResult = Result, Rc>; + pub struct Module { // URL after resolving the import specifier to an absolute URL, but without // taking into any HTTP redirects. resolved_url: Url, - source: - Shared>>>>>, + source: ModuleSourceFuture, + compiled: ModuleCompiledFuture, } +type ModuleSourceFuture = + Shared>>>>; + +type ModuleCompiledFuture = + Shared>>>>>; impl Borrow for Module { fn borrow(&self) -> &Url { &self.resolved_url @@ -47,6 +60,7 @@ impl Hash for Module { } } +#[derive(Clone)] pub struct ModuleSource { // URL that the source was actually loaded from, according to the loader. It // might differ from the specified URL for various reasons, e.g. because @@ -55,7 +69,7 @@ pub struct ModuleSource { // relative URLs, the fetched URL will be used as the basis for their // resolution. fetched_url: Url, - // JavaScript source code that is compiled by V8. + // JavaScript source code that will be compiled by V8. js_source_code: String, } @@ -69,14 +83,14 @@ impl From for ModuleSource { } pub struct ModuleImporter { - fetcher: Rc, + loader: Rc, modules: HashSet, } impl ModuleImporter { - pub fn new(fetcher: Rc) -> Self { + pub fn new(loader: Rc) -> Self { Self { - fetcher, + loader, modules: HashSet::new(), } } @@ -89,24 +103,83 @@ impl ModuleImporter { // `get_or_insert_with()` or `get_or_insert_owned()` on HashSet. It would // be more efficient to use these when they become available. if !self.modules.contains(resolved_url) { - let fetcher = self.fetcher.clone(); - let module_specifier = resolved_url.clone().into(); - let module = Module { - resolved_url: resolved_url.clone(), - source: async move { - Rc::new( - fetcher - .load(&module_specifier, None, is_dyn_import) - .map_ok(ModuleSource::from) - .await, - ) - } - .boxed_local() - .shared(), - }; + let module = Self::new_module(&self.loader, resolved_url, is_dyn_import); self.modules.insert(module); } - self.modules.get(resolved_url).unwrap() } + + pub fn new_module( + loader: &Rc, + resolved_url: &Url, + is_dyn_import: bool, + ) -> Module { + let source = + Self::new_module_source_future(loader, resolved_url, is_dyn_import); + let compiled = Self::new_module_compiled_future(loader, &source); + Module { + resolved_url: resolved_url.clone(), + source, + compiled, + } + } + + fn new_module_source_future( + loader: &Rc, + resolved_url: &Url, + is_dyn_import: bool, + ) -> ModuleSourceFuture { + let loader = loader.clone(); + let module_specifier = resolved_url.clone().into(); + + loader + .load(&module_specifier, None, is_dyn_import) + .map_ok(ModuleSource::from) + .map_ok(Rc::new) + .map_err(ImportError::Rust) + .map_err(Rc::new) + .boxed_local() + .shared() + } + + fn new_module_compiled_future( + loader: &Rc, + source: &ModuleSourceFuture, + ) -> ModuleCompiledFuture { + let loader = loader.clone(); + let source = source.clone(); + + async move { + let source = source.await?; + + #[allow(clippy::transmute_ptr_to_ptr)] + #[allow(mutable_transmutes)] + let isolate: &mut v8::OwnedIsolate = + unsafe { std::mem::transmute(loader.get_isolate()) }; + + let mut hs = v8::HandleScope::new(isolate); + let scope = hs.enter(); + + let resource_name = + v8::String::new(scope, source.fetched_url.as_str()).unwrap(); + let js_source_code = + v8::String::new(scope, &source.js_source_code).unwrap(); + + let origin = module_origin(scope, resource_name); + let source = v8::script_compiler::Source::new(js_source_code, &origin); + + let mut try_catch = v8::TryCatch::new(scope); + let tc = try_catch.enter(); + + v8::script_compiler::compile_module(scope, source) + .map(|module| v8::Global::new_from(scope, module)) + .map(Rc::new) + .ok_or_else(|| tc.exception().unwrap()) + .map_err(|exception| v8::Global::new_from(scope, exception)) + .map_err(ImportError::JS) + .map_err(Rc::new) + } + .boxed_local() + .shared() + } } diff --git a/core/modules.rs b/core/modules.rs index fbdf21b81a2af8..258cbd178e5913 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -68,6 +68,10 @@ pub trait ModuleLoader { maybe_referrer: Option, is_dyn_import: bool, ) -> Pin>; + + fn get_isolate(&self) -> &v8::OwnedIsolate { + unimplemented!(); + } } #[derive(Debug, Eq, PartialEq)] From 2978863d46eaf5c136bee302f39c22cdd2c6786c Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Tue, 10 Mar 2020 20:34:38 -0700 Subject: [PATCH 03/11] wip --- core/module_importer.rs | 72 ++++++++++++++++++++++++++++++++++------ core/module_specifier.rs | 6 ++++ 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/core/module_importer.rs b/core/module_importer.rs index 56cf453c01abf1..70cef013ce5e8b 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -1,5 +1,4 @@ use std::borrow::Borrow; -use std::collections::HashMap; use std::collections::HashSet; use std::future::Future; use std::hash::Hash; @@ -8,7 +7,6 @@ use std::pin::Pin; use std::rc::Rc; use futures::future::Shared; -use futures::future::TryFuture; use futures::FutureExt; use futures::TryFutureExt; use rusty_v8 as v8; @@ -18,7 +16,6 @@ use crate::bindings::module_origin; use crate::modules::ModuleSource as LegacyModuleSource; use crate::ErrBox; use crate::ModuleLoader; -use crate::ModuleSpecifier; pub enum ImportError { Rust(ErrBox), @@ -27,19 +24,31 @@ pub enum ImportError { pub type ImportResult = Result, Rc>; +#[derive(Clone)] pub struct Module { // URL after resolving the import specifier to an absolute URL, but without // taking into any HTTP redirects. resolved_url: Url, source: ModuleSourceFuture, - compiled: ModuleCompiledFuture, + // Future that resolves to the global v8::Module handle of the module after + // compiling the source code. This future becomes ready after the module + // has just been compiled; it may not be instantiated or evaluated yet. + handle: ModuleHandleFuture, + // Future that resolves to a Vec containing the URLs of dependencies + // imported by this module. + import_urls: ModuleImportUrlsFuture, } type ModuleSourceFuture = Shared>>>>; -type ModuleCompiledFuture = +type ModuleHandleFuture = Shared>>>>>; + +type ModuleImportUrlsFuture = futures::future::Shared< + Pin>>>>, +>; + impl Borrow for Module { fn borrow(&self) -> &Url { &self.resolved_url @@ -69,7 +78,7 @@ pub struct ModuleSource { // relative URLs, the fetched URL will be used as the basis for their // resolution. fetched_url: Url, - // JavaScript source code that will be compiled by V8. + // JavaScript source code that will be handle by V8. js_source_code: String, } @@ -116,11 +125,14 @@ impl ModuleImporter { ) -> Module { let source = Self::new_module_source_future(loader, resolved_url, is_dyn_import); - let compiled = Self::new_module_compiled_future(loader, &source); + let handle = Self::new_module_handle_future(loader, &source); + let import_urls = + Self::new_module_import_urls_future(loader, &source, &handle); Module { resolved_url: resolved_url.clone(), source, - compiled, + handle, + import_urls, } } @@ -142,10 +154,10 @@ impl ModuleImporter { .shared() } - fn new_module_compiled_future( + fn new_module_handle_future( loader: &Rc, source: &ModuleSourceFuture, - ) -> ModuleCompiledFuture { + ) -> ModuleHandleFuture { let loader = loader.clone(); let source = source.clone(); @@ -182,4 +194,44 @@ impl ModuleImporter { .boxed_local() .shared() } + + fn new_module_import_urls_future( + loader: &Rc, + source: &ModuleSourceFuture, + handle: &ModuleHandleFuture, + ) -> ModuleImportUrlsFuture { + let loader = loader.clone(); + let source = source.clone(); + let handle = handle.clone(); + + async move { + let handle = handle.await?; + let source = source.await?; + + #[allow(clippy::transmute_ptr_to_ptr)] + #[allow(mutable_transmutes)] + let isolate: &mut v8::OwnedIsolate = + unsafe { std::mem::transmute(loader.get_isolate()) }; + + let mut hs = v8::HandleScope::new(isolate); + let scope = hs.enter(); + + let handle = handle.get(scope).unwrap(); + let referrer = source.fetched_url.as_str(); + + (0..handle.get_module_requests_length()) + .map(|i| { + let specifier = + handle.get_module_request(i).to_rust_string_lossy(scope); + let url: Url = loader.resolve(&specifier, referrer, false)?.into(); + Ok(url) + }) + .collect::, ErrBox>>() + .map(Rc::new) + .map_err(ImportError::Rust) + .map_err(Rc::new) + } + .boxed_local() + .shared() + } } diff --git a/core/module_specifier.rs b/core/module_specifier.rs index 4919fdfede139c..434d890a87486b 100644 --- a/core/module_specifier.rs +++ b/core/module_specifier.rs @@ -201,6 +201,12 @@ impl From for ModuleSpecifier { } } +impl From for Url { + fn from(specifier: ModuleSpecifier) -> Self { + specifier.0 + } +} + impl PartialEq for ModuleSpecifier { fn eq(&self, other: &String) -> bool { &self.to_string() == other From 6dc07e1202c97426d0bd59345a48defd3f039ab6 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Wed, 11 Mar 2020 19:14:11 -0700 Subject: [PATCH 04/11] wip --- core/module_importer.rs | 127 +++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 35 deletions(-) diff --git a/core/module_importer.rs b/core/module_importer.rs index 70cef013ce5e8b..a1ef80ec57908f 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -1,13 +1,19 @@ use std::borrow::Borrow; +use std::cell::Ref; +use std::cell::RefCell; +use std::cell::RefMut; use std::collections::HashSet; use std::future::Future; use std::hash::Hash; use std::hash::Hasher; +use std::ops::Deref; use std::pin::Pin; use std::rc::Rc; use futures::future::Shared; +use futures::stream::FuturesUnordered; use futures::FutureExt; +use futures::StreamExt; use futures::TryFutureExt; use rusty_v8 as v8; use url::Url; @@ -36,7 +42,10 @@ pub struct Module { handle: ModuleHandleFuture, // Future that resolves to a Vec containing the URLs of dependencies // imported by this module. - import_urls: ModuleImportUrlsFuture, + deps_urls: ModuleDepsUrlsFuture, + // Future that resolves when the source for all dependencies has been fetched + // and compiled, so that their v8::Module handle is in the cache. + deps_compiled_recursively: ModuleDepsCompiledRecursivelyFuture, } type ModuleSourceFuture = @@ -45,9 +54,11 @@ type ModuleSourceFuture = type ModuleHandleFuture = Shared>>>>>; -type ModuleImportUrlsFuture = futures::future::Shared< - Pin>>>>, ->; +type ModuleDepsUrlsFuture = + Shared>>>>>; + +type ModuleDepsCompiledRecursivelyFuture = + Shared>>>>>; impl Borrow for Module { fn borrow(&self) -> &Url { @@ -91,60 +102,76 @@ impl From for ModuleSource { } } -pub struct ModuleImporter { +pub struct ModuleImporterInner { loader: Rc, modules: HashSet, } +#[derive(Clone)] +pub struct ModuleImporter(Rc>); + impl ModuleImporter { pub fn new(loader: Rc) -> Self { - Self { + Self(Rc::new(RefCell::new(ModuleImporterInner { loader, modules: HashSet::new(), - } + }))) } // The `is_dyn_import` flag is forwarded to the module loader if the module's // source hasn't been loaded yet. If the module already exists in the module // set, it is ignored. - pub fn get(&mut self, resolved_url: &Url, is_dyn_import: bool) -> &Module { + pub fn get<'a>( + &'a self, + resolved_url: &Url, + is_dyn_import: bool, + ) -> Ref<'a, Module> { // Note: future versions of Rust will likely implement a method like // `get_or_insert_with()` or `get_or_insert_owned()` on HashSet. It would // be more efficient to use these when they become available. - if !self.modules.contains(resolved_url) { - let module = Self::new_module(&self.loader, resolved_url, is_dyn_import); - self.modules.insert(module); + if self.modules().contains(resolved_url) { + let module = self.new_module(resolved_url, is_dyn_import); + self.modules().insert(module); } - self.modules.get(resolved_url).unwrap() + + Ref::map(RefCell::borrow(&self.0), |inner| { + inner.modules.get(resolved_url).unwrap() + }) } - pub fn new_module( - loader: &Rc, - resolved_url: &Url, - is_dyn_import: bool, - ) -> Module { - let source = - Self::new_module_source_future(loader, resolved_url, is_dyn_import); - let handle = Self::new_module_handle_future(loader, &source); - let import_urls = - Self::new_module_import_urls_future(loader, &source, &handle); + pub fn new_module(&self, resolved_url: &Url, is_dyn_import: bool) -> Module { + let source = self.new_module_source_future(resolved_url, is_dyn_import); + let handle = self.new_module_handle_future(&source); + let deps_urls = self.new_module_deps_urls_future(&source, &handle); + let deps_compiled_recursively = + self.new_module_deps_compiled_recursively_future(&deps_urls); + Module { resolved_url: resolved_url.clone(), source, handle, - import_urls, + deps_urls, + deps_compiled_recursively, } } + fn loader(&self) -> Ref> { + Ref::map(RefCell::borrow(&self.0), |inner| &inner.loader) + } + + fn modules(&self) -> RefMut> { + RefMut::map(RefCell::borrow_mut(&self.0), |inner| &mut inner.modules) + } + fn new_module_source_future( - loader: &Rc, + &self, resolved_url: &Url, is_dyn_import: bool, ) -> ModuleSourceFuture { - let loader = loader.clone(); let module_specifier = resolved_url.clone().into(); - loader + self + .loader() .load(&module_specifier, None, is_dyn_import) .map_ok(ModuleSource::from) .map_ok(Rc::new) @@ -155,10 +182,10 @@ impl ModuleImporter { } fn new_module_handle_future( - loader: &Rc, + &self, source: &ModuleSourceFuture, ) -> ModuleHandleFuture { - let loader = loader.clone(); + let cache = self.clone(); let source = source.clone(); async move { @@ -167,7 +194,7 @@ impl ModuleImporter { #[allow(clippy::transmute_ptr_to_ptr)] #[allow(mutable_transmutes)] let isolate: &mut v8::OwnedIsolate = - unsafe { std::mem::transmute(loader.get_isolate()) }; + unsafe { std::mem::transmute(cache.loader().get_isolate()) }; let mut hs = v8::HandleScope::new(isolate); let scope = hs.enter(); @@ -195,12 +222,12 @@ impl ModuleImporter { .shared() } - fn new_module_import_urls_future( - loader: &Rc, + fn new_module_deps_urls_future( + &self, source: &ModuleSourceFuture, handle: &ModuleHandleFuture, - ) -> ModuleImportUrlsFuture { - let loader = loader.clone(); + ) -> ModuleDepsUrlsFuture { + let cache = self.clone(); let source = source.clone(); let handle = handle.clone(); @@ -211,7 +238,7 @@ impl ModuleImporter { #[allow(clippy::transmute_ptr_to_ptr)] #[allow(mutable_transmutes)] let isolate: &mut v8::OwnedIsolate = - unsafe { std::mem::transmute(loader.get_isolate()) }; + unsafe { std::mem::transmute(cache.loader().get_isolate()) }; let mut hs = v8::HandleScope::new(isolate); let scope = hs.enter(); @@ -223,7 +250,8 @@ impl ModuleImporter { .map(|i| { let specifier = handle.get_module_request(i).to_rust_string_lossy(scope); - let url: Url = loader.resolve(&specifier, referrer, false)?.into(); + let url: Url = + cache.loader().resolve(&specifier, referrer, false)?.into(); Ok(url) }) .collect::, ErrBox>>() @@ -234,4 +262,33 @@ impl ModuleImporter { .boxed_local() .shared() } + + fn new_module_deps_compiled_recursively_future( + &self, + deps_urls: &ModuleDepsUrlsFuture, + ) -> ModuleDepsCompiledRecursivelyFuture { + let cache = self.clone(); + let deps_urls = deps_urls.clone(); + + async move { + let deps_urls = deps_urls.await?; + + let deps_compiled = deps_urls + .iter() + .map(|url| { + cache + .modules() + .get(url) + .unwrap() + .deps_compiled_recursively + .clone() + }) + .collect::>() + .fold(Ok(()), |acc, r| async move { acc.and(r) }); + + unimplemented!() + } + .boxed_local() + .shared() + } } From fc719295b8674be9f47b55b253c90afbc38feafe Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Wed, 11 Mar 2020 21:09:48 -0700 Subject: [PATCH 05/11] wip --- core/module_importer.rs | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/core/module_importer.rs b/core/module_importer.rs index a1ef80ec57908f..288c19b920e396 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -268,27 +268,23 @@ impl ModuleImporter { deps_urls: &ModuleDepsUrlsFuture, ) -> ModuleDepsCompiledRecursivelyFuture { let cache = self.clone(); - let deps_urls = deps_urls.clone(); - - async move { - let deps_urls = deps_urls.await?; - - let deps_compiled = deps_urls - .iter() - .map(|url| { - cache - .modules() - .get(url) - .unwrap() - .deps_compiled_recursively - .clone() - }) - .collect::>() - .fold(Ok(()), |acc, r| async move { acc.and(r) }); - - unimplemented!() - } - .boxed_local() - .shared() + deps_urls + .clone() + .and_then(move |dep_urls| { + dep_urls + .iter() + .map(|dep_url| { + cache + .modules() + .get(dep_url) + .unwrap() + .deps_compiled_recursively + .clone() + }) + .collect::>() + .fold(Ok(()), |acc, r| async move { acc.and(r) }) + }) + .boxed_local() + .shared() } } From 2191a3bfc91ec3b5d88b6061ec938f46fd9b4ff5 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Wed, 11 Mar 2020 21:42:54 -0700 Subject: [PATCH 06/11] fix --- cli/state.rs | 1 - core/lib.rs | 1 + core/module_importer.rs | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/state.rs b/cli/state.rs index 94dc4f5717cec7..44ab73034ec6b2 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -9,7 +9,6 @@ use crate::ops::JsonOp; use crate::ops::MinimalOp; use crate::permissions::DenoPermissions; use crate::worker::WorkerHandle; -use deno_core::v8; use deno_core::Buf; use deno_core::CoreOp; use deno_core::ErrBox; diff --git a/core/lib.rs b/core/lib.rs index e60cf55837738e..83d10f97404679 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -31,6 +31,7 @@ pub use crate::isolate::*; pub use crate::js_errors::*; pub use crate::module_importer::*; pub use crate::module_specifier::*; +pub use crate::modules::ModuleSource; pub use crate::modules::*; pub use crate::ops::*; pub use crate::plugins::*; diff --git a/core/module_importer.rs b/core/module_importer.rs index 288c19b920e396..3f525f1f6028d9 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -6,7 +6,6 @@ use std::collections::HashSet; use std::future::Future; use std::hash::Hash; use std::hash::Hasher; -use std::ops::Deref; use std::pin::Pin; use std::rc::Rc; From f95f6056e3a92ada1c6be8e4342e1af0605c65ed Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Thu, 12 Mar 2020 19:01:06 -0700 Subject: [PATCH 07/11] wip --- core/module_importer.rs | 81 +++++++++++++++++++++++++++++++++++++++++ core/modules.rs | 4 ++ 2 files changed, 85 insertions(+) diff --git a/core/module_importer.rs b/core/module_importer.rs index 3f525f1f6028d9..497248662ba012 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -3,10 +3,12 @@ use std::cell::Ref; use std::cell::RefCell; use std::cell::RefMut; use std::collections::HashSet; +use std::ffi::c_void; use std::future::Future; use std::hash::Hash; use std::hash::Hasher; use std::pin::Pin; +use std::ptr::null_mut; use std::rc::Rc; use futures::future::Shared; @@ -59,6 +61,9 @@ type ModuleDepsUrlsFuture = type ModuleDepsCompiledRecursivelyFuture = Shared>>>>>; +type ModuleEvaluationFuture = + Shared>>>>>; + impl Borrow for Module { fn borrow(&self) -> &Url { &self.resolved_url @@ -286,4 +291,80 @@ impl ModuleImporter { .boxed_local() .shared() } + + const ACTIVE_IMPORTER_KEY: u32 = 2; + + fn new_module_evaluation_future( + &self, + handle: &ModuleHandleFuture, + deps_compiled_recursively: &ModuleDepsCompiledRecursivelyFuture, + ) -> ModuleEvaluationFuture { + let cache = self.clone(); + let handle = handle.clone(); + let deps_compiled_recursively = deps_compiled_recursively.clone(); + + async { + let handle = handle.await?; + let deps_compiled_recursively = deps_compiled_recursively.await?; + + let context = cache.loader().get_context(); + #[allow(clippy::transmute_ptr_to_ptr)] + #[allow(mutable_transmutes)] + let isolate: &mut v8::OwnedIsolate = + unsafe { std::mem::transmute(cache.loader().get_isolate()) }; + let mut hs = v8::HandleScope::new(isolate); + let scope = hs.enter(); + + let handle = handle.get(scope).unwrap(); + let context = context.get(scope).unwrap(); + + unsafe { + assert_eq!( + scope.isolate().get_data(Self::ACTIVE_IMPORTER_KEY), + null_mut() + ); + scope.isolate().set_data( + Self::ACTIVE_IMPORTER_KEY, + &cache as *const _ as *mut ModuleImporter as *mut c_void, + ); + } + + let r = handle + .instantiate_module(context, Self::resolve_callback) + .unwrap(); + assert_eq!(r, true); + + unsafe { + scope + .isolate() + .set_data(Self::ACTIVE_IMPORTER_KEY, null_mut()); + } + + + unimplemented!(); + } + .boxed_local() + .shared() + } + + fn resolve_callback<'s>( + context: v8::Local<'s, v8::Context>, + specifier: v8::Local<'s, v8::String>, + referrer: v8::Local<'s, v8::Module>, + ) -> Option> { + let mut scope = v8::CallbackScope::new_escapable(context); + let mut scope = v8::EscapableHandleScope::new(scope.enter()); + let scope = scope.enter(); + + let cache = unsafe { + &*(scope.isolate().get_data(Self::ACTIVE_IMPORTER_KEY) as *const _ as *const ModuleImporter) + }; + let specifier = + specifier.to_rust_string_lossy(scope); + let referrer = referrer + let url: Url = + cache.loader().resolve(&specifier, referrer, false)?.into(); + + unimplemented!(); + } } diff --git a/core/modules.rs b/core/modules.rs index 258cbd178e5913..2612ebc8cbd9ed 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -72,6 +72,10 @@ pub trait ModuleLoader { fn get_isolate(&self) -> &v8::OwnedIsolate { unimplemented!(); } + + fn get_context(&self) -> v8::Global { + unimplemented!(); + } } #[derive(Debug, Eq, PartialEq)] From 10e14e1d7044702436fb05004fddb1ac40fb2b15 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Mon, 16 Mar 2020 18:28:21 -0700 Subject: [PATCH 08/11] wip --- Cargo.toml | 2 +- core/module_importer.rs | 34 +++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 208edf83da2d03..314ae9309a1cdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,4 @@ members = [ ] [replace] -"rusty_v8:0.3.4" = { path = 'd:/v8rs' } +"rusty_v8:0.3.6" = { path = 'd:/v8rs' } diff --git a/core/module_importer.rs b/core/module_importer.rs index 497248662ba012..642e4dd6bb6b4b 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -108,7 +108,10 @@ impl From for ModuleSource { pub struct ModuleImporterInner { loader: Rc, - modules: HashSet, + modules_by_resolved_url: HashSet, + // TODO(piscisaureus): Use a HashMap instead of a Vec. This requires rusty_v8 + // to implement Eq and Hash for `Global`. + fetched_urls_by_handle: Vec<(v8::Global, Url)>, } #[derive(Clone)] @@ -118,7 +121,8 @@ impl ModuleImporter { pub fn new(loader: Rc) -> Self { Self(Rc::new(RefCell::new(ModuleImporterInner { loader, - modules: HashSet::new(), + modules_by_resolved_url: HashSet::new(), + fetched_urls_by_handle: Vec::new(), }))) } @@ -133,13 +137,13 @@ impl ModuleImporter { // Note: future versions of Rust will likely implement a method like // `get_or_insert_with()` or `get_or_insert_owned()` on HashSet. It would // be more efficient to use these when they become available. - if self.modules().contains(resolved_url) { + if self.modules_by_resolved_url().contains(resolved_url) { let module = self.new_module(resolved_url, is_dyn_import); - self.modules().insert(module); + self.modules_by_resolved_url().insert(module); } Ref::map(RefCell::borrow(&self.0), |inner| { - inner.modules.get(resolved_url).unwrap() + inner.modules_by_resolved_url.get(resolved_url).unwrap() }) } @@ -163,8 +167,10 @@ impl ModuleImporter { Ref::map(RefCell::borrow(&self.0), |inner| &inner.loader) } - fn modules(&self) -> RefMut> { - RefMut::map(RefCell::borrow_mut(&self.0), |inner| &mut inner.modules) + fn modules_by_resolved_url(&self) -> RefMut> { + RefMut::map(RefCell::borrow_mut(&self.0), |inner| { + &mut inner.modules_by_resolved_url + }) } fn new_module_source_future( @@ -279,7 +285,7 @@ impl ModuleImporter { .iter() .map(|dep_url| { cache - .modules() + .modules_by_resolved_url() .get(dep_url) .unwrap() .deps_compiled_recursively @@ -340,7 +346,6 @@ impl ModuleImporter { .set_data(Self::ACTIVE_IMPORTER_KEY, null_mut()); } - unimplemented!(); } .boxed_local() @@ -357,13 +362,12 @@ impl ModuleImporter { let scope = scope.enter(); let cache = unsafe { - &*(scope.isolate().get_data(Self::ACTIVE_IMPORTER_KEY) as *const _ as *const ModuleImporter) + &*(scope.isolate().get_data(Self::ACTIVE_IMPORTER_KEY) as *const _ + as *const ModuleImporter) }; - let specifier = - specifier.to_rust_string_lossy(scope); - let referrer = referrer - let url: Url = - cache.loader().resolve(&specifier, referrer, false)?.into(); + let specifier = specifier.to_rust_string_lossy(scope); + let referrer = unimplemented!(); + let url: Url = cache.loader().resolve(&specifier, referrer, false)?.into(); unimplemented!(); } From 6bf75e6348f4d3be3e16651586674b0e25b434ad Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Tue, 17 Mar 2020 19:02:56 -0700 Subject: [PATCH 09/11] wip --- Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 314ae9309a1cdf..b273e4222f2448 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,3 @@ members = [ "deno_typescript", "test_plugin" ] - -[replace] -"rusty_v8:0.3.6" = { path = 'd:/v8rs' } From 3179c81711cfbaf8532c626a632cc65cb68cccc6 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Mon, 23 Mar 2020 03:24:52 +0100 Subject: [PATCH 10/11] wip --- core/module_importer.rs | 81 ++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/core/module_importer.rs b/core/module_importer.rs index 642e4dd6bb6b4b..1a6dfee1c605c1 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -47,6 +47,8 @@ pub struct Module { // Future that resolves when the source for all dependencies has been fetched // and compiled, so that their v8::Module handle is in the cache. deps_compiled_recursively: ModuleDepsCompiledRecursivelyFuture, + // Future that resolves to the result of evaluating the module. + evaluation_result: ModuleEvaluationResultFuture, } type ModuleSourceFuture = @@ -61,7 +63,7 @@ type ModuleDepsUrlsFuture = type ModuleDepsCompiledRecursivelyFuture = Shared>>>>>; -type ModuleEvaluationFuture = +type ModuleEvaluationResultFuture = Shared>>>>>; impl Borrow for Module { @@ -106,12 +108,17 @@ impl From for ModuleSource { } } +struct ModuleSourceByHandleEntry { + pub handle: Rc>, + pub source: Rc, +} + pub struct ModuleImporterInner { loader: Rc, modules_by_resolved_url: HashSet, // TODO(piscisaureus): Use a HashMap instead of a Vec. This requires rusty_v8 // to implement Eq and Hash for `Global`. - fetched_urls_by_handle: Vec<(v8::Global, Url)>, + module_sources_by_handle: Vec, } #[derive(Clone)] @@ -122,7 +129,7 @@ impl ModuleImporter { Self(Rc::new(RefCell::new(ModuleImporterInner { loader, modules_by_resolved_url: HashSet::new(), - fetched_urls_by_handle: Vec::new(), + module_sources_by_handle: Vec::new(), }))) } @@ -153,6 +160,8 @@ impl ModuleImporter { let deps_urls = self.new_module_deps_urls_future(&source, &handle); let deps_compiled_recursively = self.new_module_deps_compiled_recursively_future(&deps_urls); + let evaluation_result = self + .new_module_evaluation_result_future(&handle, &deps_compiled_recursively); Module { resolved_url: resolved_url.clone(), @@ -160,6 +169,7 @@ impl ModuleImporter { handle, deps_urls, deps_compiled_recursively, + evaluation_result, } } @@ -173,6 +183,12 @@ impl ModuleImporter { }) } + fn module_sources_by_handle(&self) -> RefMut> { + RefMut::map(RefCell::borrow_mut(&self.0), |inner| { + &mut inner.module_sources_by_handle + }) + } + fn new_module_source_future( &self, resolved_url: &Url, @@ -214,19 +230,29 @@ impl ModuleImporter { let js_source_code = v8::String::new(scope, &source.js_source_code).unwrap(); - let origin = module_origin(scope, resource_name); - let source = v8::script_compiler::Source::new(js_source_code, &origin); + let v8_origin = module_origin(scope, resource_name); + let v8_source = + v8::script_compiler::Source::new(js_source_code, &v8_origin); let mut try_catch = v8::TryCatch::new(scope); let tc = try_catch.enter(); - v8::script_compiler::compile_module(scope, source) - .map(|module| v8::Global::new_from(scope, module)) + let handle = v8::script_compiler::compile_module(scope, v8_source) + .map(|handle| v8::Global::new_from(scope, handle)) .map(Rc::new) .ok_or_else(|| tc.exception().unwrap()) .map_err(|exception| v8::Global::new_from(scope, exception)) .map_err(ImportError::JS) - .map_err(Rc::new) + .map_err(Rc::new)?; + + cache + .module_sources_by_handle() + .push(ModuleSourceByHandleEntry { + handle: handle.clone(), + source, + }); + + Ok(handle) } .boxed_local() .shared() @@ -300,18 +326,18 @@ impl ModuleImporter { const ACTIVE_IMPORTER_KEY: u32 = 2; - fn new_module_evaluation_future( + fn new_module_evaluation_result_future( &self, handle: &ModuleHandleFuture, deps_compiled_recursively: &ModuleDepsCompiledRecursivelyFuture, - ) -> ModuleEvaluationFuture { + ) -> ModuleEvaluationResultFuture { let cache = self.clone(); let handle = handle.clone(); let deps_compiled_recursively = deps_compiled_recursively.clone(); - async { + async move { let handle = handle.await?; - let deps_compiled_recursively = deps_compiled_recursively.await?; + deps_compiled_recursively.await?; let context = cache.loader().get_context(); #[allow(clippy::transmute_ptr_to_ptr)] @@ -321,7 +347,7 @@ impl ModuleImporter { let mut hs = v8::HandleScope::new(isolate); let scope = hs.enter(); - let handle = handle.get(scope).unwrap(); + let mut handle = handle.get(scope).unwrap(); let context = context.get(scope).unwrap(); unsafe { @@ -365,10 +391,31 @@ impl ModuleImporter { &*(scope.isolate().get_data(Self::ACTIVE_IMPORTER_KEY) as *const _ as *const ModuleImporter) }; - let specifier = specifier.to_rust_string_lossy(scope); - let referrer = unimplemented!(); - let url: Url = cache.loader().resolve(&specifier, referrer, false)?.into(); - unimplemented!(); + let specifier_str = &specifier.to_rust_string_lossy(scope); + + let referrer_source = cache + .module_sources_by_handle() + .iter() + .find(|entry| entry.handle.get(scope).unwrap() == referrer) + .map(|entry| entry.source.clone()) + .unwrap(); + let referrer_str = referrer_source.fetched_url.as_str(); + + let resolved_url: Url = cache + .loader() + .resolve(specifier_str, referrer_str, false) + .unwrap() + .into(); + + let handle = cache + .modules_by_resolved_url() + .get(&resolved_url) + .and_then(|module| module.handle.peek()) + .and_then(|result| result.clone().ok()) + .and_then(|handle| handle.get(scope)) + .unwrap(); + + Some(scope.escape(handle)) } } From 5fc47b96a517f12f049abf237b3c5e31580ce473 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Mon, 23 Mar 2020 03:55:03 +0100 Subject: [PATCH 11/11] nearing completion --- core/module_importer.rs | 79 +++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/core/module_importer.rs b/core/module_importer.rs index 1a6dfee1c605c1..15f6b6f9088b31 100644 --- a/core/module_importer.rs +++ b/core/module_importer.rs @@ -29,8 +29,38 @@ pub enum ImportError { JS(v8::Global), } +impl ImportError { + pub fn as_js_exception<'s>( + &self, + scope: &mut impl v8::ToLocal<'s>, + ) -> v8::Local<'s, v8::Value> { + match self { + Self::Rust(err) => { + let message = v8::String::new(scope, &err.to_string()).unwrap(); + v8::Exception::type_error(scope, message) + } + Self::JS(handle) => handle.get(scope).unwrap(), + } + } +} + pub type ImportResult = Result, Rc>; +type ModuleSourceFuture = + Shared>>>>; + +type ModuleHandleFuture = + Shared>>>>>; + +type ModuleDepsUrlsFuture = + Shared>>>>>; + +type ModuleDepsCompiledRecursivelyFuture = + Shared>>>>>; + +type ModuleEvaluationResultFuture = + Shared>>>>>; + #[derive(Clone)] pub struct Module { // URL after resolving the import specifier to an absolute URL, but without @@ -51,21 +81,6 @@ pub struct Module { evaluation_result: ModuleEvaluationResultFuture, } -type ModuleSourceFuture = - Shared>>>>; - -type ModuleHandleFuture = - Shared>>>>>; - -type ModuleDepsUrlsFuture = - Shared>>>>>; - -type ModuleDepsCompiledRecursivelyFuture = - Shared>>>>>; - -type ModuleEvaluationResultFuture = - Shared>>>>>; - impl Borrow for Module { fn borrow(&self) -> &Url { &self.resolved_url @@ -136,7 +151,7 @@ impl ModuleImporter { // The `is_dyn_import` flag is forwarded to the module loader if the module's // source hasn't been loaded yet. If the module already exists in the module // set, it is ignored. - pub fn get<'a>( + pub fn get_or_create_module<'a>( &'a self, resolved_url: &Url, is_dyn_import: bool, @@ -145,7 +160,7 @@ impl ModuleImporter { // `get_or_insert_with()` or `get_or_insert_owned()` on HashSet. It would // be more efficient to use these when they become available. if self.modules_by_resolved_url().contains(resolved_url) { - let module = self.new_module(resolved_url, is_dyn_import); + let module = self.create_module(resolved_url, is_dyn_import); self.modules_by_resolved_url().insert(module); } @@ -154,17 +169,21 @@ impl ModuleImporter { }) } - pub fn new_module(&self, resolved_url: &Url, is_dyn_import: bool) -> Module { - let source = self.new_module_source_future(resolved_url, is_dyn_import); - let handle = self.new_module_handle_future(&source); - let deps_urls = self.new_module_deps_urls_future(&source, &handle); + fn create_module(&self, resolved_url: &Url, is_dyn_import: bool) -> Module { + let resolved_url = resolved_url.clone(); + + let source = self.create_module_source_future(&resolved_url, is_dyn_import); + let handle = self.create_module_handle_future(&source); + let deps_urls = self.create_module_deps_urls_future(&source, &handle); let deps_compiled_recursively = - self.new_module_deps_compiled_recursively_future(&deps_urls); - let evaluation_result = self - .new_module_evaluation_result_future(&handle, &deps_compiled_recursively); + self.create_module_deps_compiled_recursively_future(&deps_urls); + let evaluation_result = self.create_module_evaluation_result_future( + &handle, + &deps_compiled_recursively, + ); Module { - resolved_url: resolved_url.clone(), + resolved_url, source, handle, deps_urls, @@ -189,7 +208,7 @@ impl ModuleImporter { }) } - fn new_module_source_future( + fn create_module_source_future( &self, resolved_url: &Url, is_dyn_import: bool, @@ -207,7 +226,7 @@ impl ModuleImporter { .shared() } - fn new_module_handle_future( + fn create_module_handle_future( &self, source: &ModuleSourceFuture, ) -> ModuleHandleFuture { @@ -258,7 +277,7 @@ impl ModuleImporter { .shared() } - fn new_module_deps_urls_future( + fn create_module_deps_urls_future( &self, source: &ModuleSourceFuture, handle: &ModuleHandleFuture, @@ -299,7 +318,7 @@ impl ModuleImporter { .shared() } - fn new_module_deps_compiled_recursively_future( + fn create_module_deps_compiled_recursively_future( &self, deps_urls: &ModuleDepsUrlsFuture, ) -> ModuleDepsCompiledRecursivelyFuture { @@ -326,7 +345,7 @@ impl ModuleImporter { const ACTIVE_IMPORTER_KEY: u32 = 2; - fn new_module_evaluation_result_future( + fn create_module_evaluation_result_future( &self, handle: &ModuleHandleFuture, deps_compiled_recursively: &ModuleDepsCompiledRecursivelyFuture,