From 73efed218fe94ab27e89675ee238e2933b3fd4d1 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 13 Sep 2022 11:59:01 -0400 Subject: [PATCH] refactor(npm): create general use `NpmPackageResolver` (#15882) --- cli/main.rs | 2 +- cli/module_loader.rs | 1 - cli/node/mod.rs | 19 +- cli/npm/cache.rs | 2 +- cli/npm/mod.rs | 416 +----------------------------------- cli/npm/registry.rs | 15 -- cli/npm/resolution.rs | 2 + cli/npm/resolvers/common.rs | 89 ++++++++ cli/npm/resolvers/global.rs | 138 ++++++++++++ cli/npm/resolvers/mod.rs | 158 ++++++++++++++ cli/proc_state.rs | 24 ++- cli/worker.rs | 1 - ext/node/lib.rs | 20 +- ext/node/package_json.rs | 4 +- ext/node/resolution.rs | 18 +- runtime/web_worker.rs | 4 +- runtime/worker.rs | 4 +- 17 files changed, 445 insertions(+), 472 deletions(-) create mode 100644 cli/npm/resolvers/common.rs create mode 100644 cli/npm/resolvers/global.rs create mode 100644 cli/npm/resolvers/mod.rs diff --git a/cli/main.rs b/cli/main.rs index 1c9c10281e6d09..70c8c78d085ebb 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -133,7 +133,7 @@ fn print_cache_info( ) -> Result<(), AnyError> { let deno_dir = &state.dir.root; let modules_cache = &state.file_fetcher.get_http_cache_location(); - let npm_cache = &state.npm_resolver.get_cache_location(); + let npm_cache = &state.npm_cache.as_readonly().get_cache_location(); let typescript_cache = &state.dir.gen_cache.location; let registry_cache = &state.dir.root.join(lsp::language_server::REGISTRIES_PATH); diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 2a7a69dca99a94..1213358b9770f4 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -4,7 +4,6 @@ use crate::emit::emit_parsed_source; use crate::emit::TsTypeLib; use crate::graph_util::ModuleEntry; use crate::node; -use crate::npm::NpmPackageResolver; use crate::proc_state::ProcState; use crate::text_encoding::code_without_source_map; use crate::text_encoding::source_map_from_code; diff --git a/cli/node/mod.rs b/cli/node/mod.rs index cd803cc4f363cf..7e1ea95f7bb2d1 100644 --- a/cli/node/mod.rs +++ b/cli/node/mod.rs @@ -24,9 +24,9 @@ use deno_runtime::deno_node::legacy_main_resolve; use deno_runtime::deno_node::package_exports_resolve; use deno_runtime::deno_node::package_imports_resolve; use deno_runtime::deno_node::package_resolve; -use deno_runtime::deno_node::DenoDirNpmResolver; use deno_runtime::deno_node::NodeModuleKind; use deno_runtime::deno_node::PackageJson; +use deno_runtime::deno_node::RequireNpmResolver; use deno_runtime::deno_node::DEFAULT_CONDITIONS; use deno_runtime::deno_node::NODE_GLOBAL_THIS_NAME; use once_cell::sync::Lazy; @@ -34,7 +34,6 @@ use path_clean::PathClean; use regex::Regex; use crate::file_fetcher::FileFetcher; -use crate::npm::GlobalNpmPackageResolver; use crate::npm::NpmPackageReference; use crate::npm::NpmPackageReq; use crate::npm::NpmPackageResolver; @@ -380,7 +379,7 @@ pub async fn initialize_binary_command( pub fn node_resolve( specifier: &str, referrer: &ModuleSpecifier, - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result, AnyError> { // Note: if we are here, then the referrer is an esm module // TODO(bartlomieju): skipped "policy" part as we don't plan to support it @@ -431,7 +430,7 @@ pub fn node_resolve( pub fn node_resolve_npm_reference( reference: &NpmPackageReference, - npm_resolver: &GlobalNpmPackageResolver, + npm_resolver: &NpmPackageResolver, ) -> Result, AnyError> { let package_folder = npm_resolver .resolve_package_from_deno_module(&reference.req)? @@ -460,7 +459,7 @@ pub fn node_resolve_npm_reference( pub fn node_resolve_binary_export( pkg_req: &NpmPackageReq, bin_name: Option<&str>, - npm_resolver: &GlobalNpmPackageResolver, + npm_resolver: &NpmPackageResolver, ) -> Result { let pkg = npm_resolver.resolve_package_from_deno_module(pkg_req)?; let package_folder = pkg.folder_path; @@ -541,7 +540,7 @@ pub fn load_cjs_module_from_ext_node( fn package_config_resolve( package_subpath: &str, package_dir: &Path, - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, referrer_kind: NodeModuleKind, ) -> Result { let package_json_path = package_dir.join("package.json"); @@ -568,7 +567,7 @@ fn package_config_resolve( fn url_to_node_resolution( url: ModuleSpecifier, - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { Ok(if url.as_str().starts_with("http") { NodeResolution::Esm(url) @@ -640,7 +639,7 @@ fn module_resolve( specifier: &str, referrer: &ModuleSpecifier, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result, AnyError> { // note: if we're here, the referrer is an esm module let url = if should_be_treated_as_relative_or_absolute_path(specifier) { @@ -721,7 +720,7 @@ pub fn translate_cjs_to_esm( specifier: &ModuleSpecifier, code: String, media_type: MediaType, - npm_resolver: &GlobalNpmPackageResolver, + npm_resolver: &NpmPackageResolver, ) -> Result { fn perform_cjs_analysis( specifier: &str, @@ -843,7 +842,7 @@ fn resolve( specifier: &str, referrer: &ModuleSpecifier, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { if specifier.starts_with('/') { todo!(); diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index a0082fb29a0ab6..71a82fd51eab8b 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -18,10 +18,10 @@ use crate::file_fetcher::CacheSetting; use crate::fs_util; use crate::progress_bar::ProgressBar; +use super::registry::NpmPackageVersionDistInfo; use super::semver::NpmVersion; use super::tarball::verify_and_extract_tarball; use super::NpmPackageId; -use super::NpmPackageVersionDistInfo; pub const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock"; diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 2dd871cffe9e6c..d60c06f144c9ec 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -3,422 +3,14 @@ mod cache; mod registry; mod resolution; +mod resolvers; mod semver; mod tarball; -use std::io::ErrorKind; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; - -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; -use deno_core::anyhow::Context; -use deno_core::error::custom_error; -use deno_core::error::AnyError; -use deno_core::futures; -use deno_core::url::Url; -use deno_runtime::deno_node::DenoDirNpmResolver; +pub use cache::NpmCache; +pub use registry::NpmRegistryApi; pub use resolution::NpmPackageId; pub use resolution::NpmPackageReference; pub use resolution::NpmPackageReq; pub use resolution::NpmResolutionPackage; - -use cache::NpmCache; -use registry::NpmPackageVersionDistInfo; -use registry::NpmRegistryApi; -use resolution::NpmResolution; - -use crate::deno_dir::DenoDir; -use crate::file_fetcher::CacheSetting; -use crate::progress_bar::ProgressBar; - -use self::cache::ReadonlyNpmCache; -use self::resolution::NpmResolutionSnapshot; - -/// Information about the local npm package. -pub struct LocalNpmPackageInfo { - /// Unique identifier. - pub id: NpmPackageId, - /// Local folder path of the npm package. - pub folder_path: PathBuf, -} - -pub trait NpmPackageResolver { - /// Resolves an npm package from a Deno module. - fn resolve_package_from_deno_module( - &self, - pkg_req: &NpmPackageReq, - ) -> Result; - - /// Resolves an npm package from an npm package referrer. - fn resolve_package_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result; - - /// Resolve the root folder of the package the provided specifier is in. - /// - /// This will error when the provided specifier is not in an npm package. - fn resolve_package_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result; - - /// Gets if the provided specifier is in an npm package. - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - self.resolve_package_from_specifier(specifier).is_ok() - } -} - -#[derive(Debug, Clone)] -pub struct GlobalNpmPackageResolver { - cache: NpmCache, - resolution: Arc, - registry_url: Url, - unstable: bool, - no_npm: bool, -} - -impl GlobalNpmPackageResolver { - pub fn from_deno_dir( - dir: &DenoDir, - reload: bool, - cache_setting: CacheSetting, - unstable: bool, - no_npm: bool, - progress_bar: ProgressBar, - ) -> Self { - Self::from_cache( - NpmCache::from_deno_dir(dir, cache_setting.clone(), progress_bar.clone()), - reload, - cache_setting, - unstable, - no_npm, - progress_bar, - ) - } - - fn from_cache( - cache: NpmCache, - reload: bool, - cache_setting: CacheSetting, - unstable: bool, - no_npm: bool, - progress_bar: ProgressBar, - ) -> Self { - let api = - NpmRegistryApi::new(cache.clone(), reload, cache_setting, progress_bar); - let registry_url = api.base_url().to_owned(); - let resolution = Arc::new(NpmResolution::new(api)); - - Self { - cache, - resolution, - registry_url, - unstable, - no_npm, - } - } - - /// If the resolver has resolved any npm packages. - pub fn has_packages(&self) -> bool { - self.resolution.has_packages() - } - - /// Adds a package requirement to the resolver. - pub async fn add_package_reqs( - &self, - packages: Vec, - ) -> Result<(), AnyError> { - assert!(!packages.is_empty()); - - if !self.unstable { - bail!( - "Unstable use of npm specifiers. The --unstable flag must be provided." - ) - } - - if self.no_npm { - let fmt_reqs = packages - .iter() - .map(|p| format!("\"{}\"", p)) - .collect::>() - .join(", "); - return Err(custom_error( - "NoNpm", - format!( - "Following npm specifiers were requested: {}; but --no-npm is specified.", - fmt_reqs - ), - )); - } - self.resolution.add_package_reqs(packages).await - } - - /// Caches all the packages in parallel. - pub async fn cache_packages(&self) -> Result<(), AnyError> { - if std::env::var("DENO_UNSTABLE_NPM_SYNC_DOWNLOAD") == Ok("1".to_string()) { - // for some of the tests, we want downloading of packages - // to be deterministic so that the output is always the same - let mut packages = self.resolution.all_packages(); - packages.sort_by(|a, b| a.id.cmp(&b.id)); - for package in packages { - self - .cache - .ensure_package(&package.id, &package.dist, &self.registry_url) - .await - .with_context(|| { - format!("Failed caching npm package '{}'.", package.id) - })?; - } - } else { - let handles = self.resolution.all_packages().into_iter().map(|package| { - let cache = self.cache.clone(); - let registry_url = self.registry_url.clone(); - tokio::task::spawn(async move { - cache - .ensure_package(&package.id, &package.dist, ®istry_url) - .await - .with_context(|| { - format!("Failed caching npm package '{}'.", package.id) - }) - }) - }); - let results = futures::future::join_all(handles).await; - for result in results { - // surface the first error - result??; - } - } - Ok(()) - } - - fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo { - LocalNpmPackageInfo { - folder_path: self.cache.package_folder(id, &self.registry_url), - id: id.clone(), - } - } - - /// Creates an inner clone. - #[allow(unused)] - pub fn snapshot(&self) -> NpmPackageResolverSnapshot { - NpmPackageResolverSnapshot { - cache: self.cache.as_readonly(), - snapshot: self.resolution.snapshot(), - registry_url: self.registry_url.clone(), - } - } - - pub fn get_cache_location(&self) -> PathBuf { - self.cache.as_readonly().get_cache_location() - } -} - -impl NpmPackageResolver for GlobalNpmPackageResolver { - fn resolve_package_from_deno_module( - &self, - pkg_req: &NpmPackageReq, - ) -> Result { - let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?; - Ok(self.local_package_info(&pkg.id)) - } - - fn resolve_package_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result { - let referrer_pkg_id = self - .cache - .resolve_package_id_from_specifier(referrer, &self.registry_url)?; - let pkg = self - .resolution - .resolve_package_from_package(name, &referrer_pkg_id)?; - Ok(self.local_package_info(&pkg.id)) - } - - fn resolve_package_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result { - let pkg_id = self - .cache - .resolve_package_id_from_specifier(specifier, &self.registry_url)?; - Ok(self.local_package_info(&pkg_id)) - } -} - -#[derive(Clone, Debug)] -pub struct NpmPackageResolverSnapshot { - cache: ReadonlyNpmCache, - snapshot: NpmResolutionSnapshot, - registry_url: Url, -} - -// todo(dsherret): implementing Default for this is error prone, but -// necessary for the LSP. We should remove this Default implementation. -// See comment on `ReadonlyNpmCache` for more details. -impl Default for NpmPackageResolverSnapshot { - fn default() -> Self { - Self { - cache: Default::default(), - snapshot: Default::default(), - registry_url: NpmRegistryApi::default_url(), - } - } -} - -impl NpmPackageResolverSnapshot { - fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo { - LocalNpmPackageInfo { - folder_path: self.cache.package_folder(id, &self.registry_url), - id: id.clone(), - } - } -} - -impl NpmPackageResolver for NpmPackageResolverSnapshot { - fn resolve_package_from_deno_module( - &self, - pkg_req: &NpmPackageReq, - ) -> Result { - let pkg = self.snapshot.resolve_package_from_deno_module(pkg_req)?; - Ok(self.local_package_info(&pkg.id)) - } - - fn resolve_package_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result { - let referrer_pkg_id = self - .cache - .resolve_package_id_from_specifier(referrer, &self.registry_url)?; - let pkg = self - .snapshot - .resolve_package_from_package(name, &referrer_pkg_id)?; - Ok(self.local_package_info(&pkg.id)) - } - - fn resolve_package_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result { - let pkg_id = self - .cache - .resolve_package_id_from_specifier(specifier, &self.registry_url)?; - Ok(self.local_package_info(&pkg_id)) - } -} - -impl DenoDirNpmResolver for GlobalNpmPackageResolver { - fn resolve_package_folder_from_package( - &self, - specifier: &str, - referrer: &std::path::Path, - ) -> Result { - let referrer = specifier_to_path(referrer)?; - self - .resolve_package_from_package(specifier, &referrer) - .map(|p| p.folder_path) - } - - fn resolve_package_folder_from_path( - &self, - path: &Path, - ) -> Result { - let specifier = specifier_to_path(path)?; - self - .resolve_package_from_specifier(&specifier) - .map(|p| p.folder_path) - } - - fn in_npm_package(&self, path: &Path) -> bool { - let specifier = match ModuleSpecifier::from_file_path(path) { - Ok(p) => p, - Err(_) => return false, - }; - self.resolve_package_from_specifier(&specifier).is_ok() - } - - fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { - let registry_path = self.cache.registry_folder(&self.registry_url); - ensure_read_permission(®istry_path, path) - } -} - -impl DenoDirNpmResolver for NpmPackageResolverSnapshot { - fn resolve_package_folder_from_package( - &self, - specifier: &str, - referrer: &std::path::Path, - ) -> Result { - let referrer = specifier_to_path(referrer)?; - self - .resolve_package_from_package(specifier, &referrer) - .map(|p| p.folder_path) - } - - fn resolve_package_folder_from_path( - &self, - path: &Path, - ) -> Result { - let specifier = specifier_to_path(path)?; - self - .resolve_package_from_specifier(&specifier) - .map(|p| p.folder_path) - } - - fn in_npm_package(&self, path: &Path) -> bool { - let specifier = match ModuleSpecifier::from_file_path(path) { - Ok(p) => p, - Err(_) => return false, - }; - self.resolve_package_from_specifier(&specifier).is_ok() - } - - fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { - let registry_path = self.cache.registry_folder(&self.registry_url); - ensure_read_permission(®istry_path, path) - } -} - -fn specifier_to_path(path: &Path) -> Result { - match ModuleSpecifier::from_file_path(&path) { - Ok(specifier) => Ok(specifier), - Err(()) => bail!("Could not convert '{}' to url.", path.display()), - } -} - -fn ensure_read_permission( - registry_path: &Path, - path: &Path, -) -> Result<(), AnyError> { - // allow reading if it's in the deno_dir node modules - if path.starts_with(®istry_path) - && path - .components() - .all(|c| !matches!(c, std::path::Component::ParentDir)) - { - // todo(dsherret): cache this? - if let Ok(registry_path) = std::fs::canonicalize(registry_path) { - match std::fs::canonicalize(path) { - Ok(path) if path.starts_with(registry_path) => { - return Ok(()); - } - Err(e) if e.kind() == ErrorKind::NotFound => { - return Ok(()); - } - _ => {} // ignore - } - } - } - - Err(deno_core::error::custom_error( - "PermissionDenied", - format!("Reading {} is not allowed", path.display()), - )) -} +pub use resolvers::NpmPackageResolver; diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index f5b8b9fc80dcd6..4fb926287f5b22 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -131,21 +131,6 @@ impl NpmRegistryApi { } pub fn new( - cache: NpmCache, - reload: bool, - cache_setting: CacheSetting, - progress_bar: ProgressBar, - ) -> Self { - Self::from_base( - Self::default_url(), - cache, - reload, - cache_setting, - progress_bar, - ) - } - - pub fn from_base( base_url: Url, cache: NpmCache, reload: bool, diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index 8fac92716efbc3..4bde3b34065fed 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -471,6 +471,8 @@ impl NpmResolution { !self.snapshot.read().packages.is_empty() } + // todo(dsherret): for use in the lsp + #[allow(dead_code)] pub fn snapshot(&self) -> NpmResolutionSnapshot { self.snapshot.read().clone() } diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs new file mode 100644 index 00000000000000..f0231859a0a17e --- /dev/null +++ b/cli/npm/resolvers/common.rs @@ -0,0 +1,89 @@ +use std::path::Path; +use std::path::PathBuf; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::futures::future::BoxFuture; +use deno_core::url::Url; + +use crate::npm::NpmCache; +use crate::npm::NpmPackageId; +use crate::npm::NpmPackageReq; +use crate::npm::NpmResolutionPackage; + +/// Information about the local npm package. +pub struct LocalNpmPackageInfo { + /// Unique identifier. + pub id: NpmPackageId, + /// Local folder path of the npm package. + pub folder_path: PathBuf, +} + +pub trait InnerNpmPackageResolver: Send + Sync { + fn resolve_package_from_deno_module( + &self, + pkg_req: &NpmPackageReq, + ) -> Result; + + fn resolve_package_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + ) -> Result; + + fn resolve_package_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result; + + fn has_packages(&self) -> bool; + + fn add_package_reqs( + &self, + packages: Vec, + ) -> BoxFuture<'static, Result<(), AnyError>>; + + fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError>; +} + +/// Caches all the packages in parallel. +pub async fn cache_packages( + mut packages: Vec, + cache: &NpmCache, + registry_url: &Url, +) -> Result<(), AnyError> { + if std::env::var("DENO_UNSTABLE_NPM_SYNC_DOWNLOAD") == Ok("1".to_string()) { + // for some of the tests, we want downloading of packages + // to be deterministic so that the output is always the same + packages.sort_by(|a, b| a.id.cmp(&b.id)); + for package in packages { + cache + .ensure_package(&package.id, &package.dist, registry_url) + .await + .with_context(|| { + format!("Failed caching npm package '{}'.", package.id) + })?; + } + } else { + let handles = packages.into_iter().map(|package| { + let cache = cache.clone(); + let registry_url = registry_url.clone(); + tokio::task::spawn(async move { + cache + .ensure_package(&package.id, &package.dist, ®istry_url) + .await + .with_context(|| { + format!("Failed caching npm package '{}'.", package.id) + }) + }) + }); + let results = futures::future::join_all(handles).await; + for result in results { + // surface the first error + result??; + } + } + Ok(()) +} diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs new file mode 100644 index 00000000000000..259d9b9a04ef18 --- /dev/null +++ b/cli/npm/resolvers/global.rs @@ -0,0 +1,138 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::io::ErrorKind; +use std::path::Path; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; +use deno_core::futures::future::BoxFuture; +use deno_core::futures::FutureExt; +use deno_core::url::Url; + +use crate::npm::resolution::NpmResolution; +use crate::npm::resolvers::common::cache_packages; +use crate::npm::NpmCache; +use crate::npm::NpmPackageId; +use crate::npm::NpmPackageReq; +use crate::npm::NpmRegistryApi; + +use super::common::InnerNpmPackageResolver; +use super::common::LocalNpmPackageInfo; + +#[derive(Debug, Clone)] +pub struct GlobalNpmPackageResolver { + cache: NpmCache, + resolution: Arc, + registry_url: Url, +} + +impl GlobalNpmPackageResolver { + pub fn new(cache: NpmCache, api: NpmRegistryApi) -> Self { + let registry_url = api.base_url().to_owned(); + let resolution = Arc::new(NpmResolution::new(api)); + + Self { + cache, + resolution, + registry_url, + } + } + + fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo { + LocalNpmPackageInfo { + folder_path: self.cache.package_folder(id, &self.registry_url), + id: id.clone(), + } + } +} + +impl InnerNpmPackageResolver for GlobalNpmPackageResolver { + fn resolve_package_from_deno_module( + &self, + pkg_req: &NpmPackageReq, + ) -> Result { + let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?; + Ok(self.local_package_info(&pkg.id)) + } + + fn resolve_package_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + ) -> Result { + let referrer_pkg_id = self + .cache + .resolve_package_id_from_specifier(referrer, &self.registry_url)?; + let pkg = self + .resolution + .resolve_package_from_package(name, &referrer_pkg_id)?; + Ok(self.local_package_info(&pkg.id)) + } + + fn resolve_package_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + let pkg_id = self + .cache + .resolve_package_id_from_specifier(specifier, &self.registry_url)?; + Ok(self.local_package_info(&pkg_id)) + } + + fn has_packages(&self) -> bool { + self.resolution.has_packages() + } + + fn add_package_reqs( + &self, + packages: Vec, + ) -> BoxFuture<'static, Result<(), AnyError>> { + let resolver = self.clone(); + async move { + resolver.resolution.add_package_reqs(packages).await?; + cache_packages( + resolver.resolution.all_packages(), + &resolver.cache, + &resolver.registry_url, + ) + .await + } + .boxed() + } + + fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { + let registry_path = self.cache.registry_folder(&self.registry_url); + ensure_read_permission(®istry_path, path) + } +} + +fn ensure_read_permission( + registry_path: &Path, + path: &Path, +) -> Result<(), AnyError> { + // allow reading if it's in the deno_dir node modules + if path.starts_with(®istry_path) + && path + .components() + .all(|c| !matches!(c, std::path::Component::ParentDir)) + { + // todo(dsherret): cache this? + if let Ok(registry_path) = std::fs::canonicalize(registry_path) { + match std::fs::canonicalize(path) { + Ok(path) if path.starts_with(registry_path) => { + return Ok(()); + } + Err(e) if e.kind() == ErrorKind::NotFound => { + return Ok(()); + } + _ => {} // ignore + } + } + } + + Err(deno_core::error::custom_error( + "PermissionDenied", + format!("Reading {} is not allowed", path.display()), + )) +} diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs new file mode 100644 index 00000000000000..02e5be983d87ac --- /dev/null +++ b/cli/npm/resolvers/mod.rs @@ -0,0 +1,158 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +mod common; +mod global; + +use deno_core::anyhow::bail; +use deno_core::error::custom_error; +use deno_runtime::deno_node::RequireNpmResolver; +use global::GlobalNpmPackageResolver; + +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; + +use self::common::InnerNpmPackageResolver; +use super::NpmCache; +use super::NpmPackageReq; +use super::NpmRegistryApi; + +pub use self::common::LocalNpmPackageInfo; + +#[derive(Clone)] +pub struct NpmPackageResolver { + unstable: bool, + no_npm: bool, + inner: Arc, +} + +impl NpmPackageResolver { + pub fn new( + cache: NpmCache, + api: NpmRegistryApi, + unstable: bool, + no_npm: bool, + ) -> Self { + // For now, always create a GlobalNpmPackageResolver, but in the future + // this might be a local node_modules folder + let inner = Arc::new(GlobalNpmPackageResolver::new(cache, api)); + Self { + unstable, + no_npm, + inner, + } + } + + /// Resolves an npm package from a Deno module. + pub fn resolve_package_from_deno_module( + &self, + pkg_req: &NpmPackageReq, + ) -> Result { + self.inner.resolve_package_from_deno_module(pkg_req) + } + + /// Resolves an npm package from an npm package referrer. + pub fn resolve_package_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + ) -> Result { + self.inner.resolve_package_from_package(name, referrer) + } + + /// Resolve the root folder of the package the provided specifier is in. + /// + /// This will error when the provided specifier is not in an npm package. + pub fn resolve_package_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + self.inner.resolve_package_from_specifier(specifier) + } + + /// Gets if the provided specifier is in an npm package. + pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { + self.resolve_package_from_specifier(specifier).is_ok() + } + + /// If the resolver has resolved any npm packages. + pub fn has_packages(&self) -> bool { + self.inner.has_packages() + } + + /// Adds a package requirement to the resolver and ensures everything is setup. + pub async fn add_package_reqs( + &self, + packages: Vec, + ) -> Result<(), AnyError> { + assert!(!packages.is_empty()); + + if !self.unstable { + bail!( + "Unstable use of npm specifiers. The --unstable flag must be provided." + ) + } + + if self.no_npm { + let fmt_reqs = packages + .iter() + .map(|p| format!("\"{}\"", p)) + .collect::>() + .join(", "); + return Err(custom_error( + "NoNpm", + format!( + "Following npm specifiers were requested: {}; but --no-npm is specified.", + fmt_reqs + ), + )); + } + + self.inner.add_package_reqs(packages).await + } +} + +impl RequireNpmResolver for NpmPackageResolver { + fn resolve_package_folder_from_package( + &self, + specifier: &str, + referrer: &std::path::Path, + ) -> Result { + let referrer = specifier_to_path(referrer)?; + self + .resolve_package_from_package(specifier, &referrer) + .map(|p| p.folder_path) + } + + fn resolve_package_folder_from_path( + &self, + path: &Path, + ) -> Result { + let specifier = specifier_to_path(path)?; + self + .resolve_package_from_specifier(&specifier) + .map(|p| p.folder_path) + } + + fn in_npm_package(&self, path: &Path) -> bool { + let specifier = match ModuleSpecifier::from_file_path(path) { + Ok(p) => p, + Err(_) => return false, + }; + self.resolve_package_from_specifier(&specifier).is_ok() + } + + fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { + self.inner.ensure_read_permission(path) + } +} + +fn specifier_to_path(path: &Path) -> Result { + match ModuleSpecifier::from_file_path(&path) { + Ok(specifier) => Ok(specifier), + Err(()) => bail!("Could not convert '{}' to url.", path.display()), + } +} diff --git a/cli/proc_state.rs b/cli/proc_state.rs index f19132c8b2d6d2..2d2c3a76de6d85 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -22,9 +22,10 @@ use crate::lockfile::as_maybe_locker; use crate::lockfile::Lockfile; use crate::node; use crate::node::NodeResolution; -use crate::npm::GlobalNpmPackageResolver; +use crate::npm::NpmCache; use crate::npm::NpmPackageReference; use crate::npm::NpmPackageResolver; +use crate::npm::NpmRegistryApi; use crate::progress_bar::ProgressBar; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; @@ -87,7 +88,8 @@ pub struct Inner { pub parsed_source_cache: ParsedSourceCache, maybe_resolver: Option>, maybe_file_watcher_reporter: Option, - pub npm_resolver: GlobalNpmPackageResolver, + pub npm_cache: NpmCache, + pub npm_resolver: NpmPackageResolver, pub cjs_resolutions: Mutex>, progress_bar: ProgressBar, } @@ -220,15 +222,26 @@ impl ProcState { let emit_cache = EmitCache::new(dir.gen_cache.clone()); let parsed_source_cache = ParsedSourceCache::new(Some(dir.dep_analysis_db_file_path())); - let npm_resolver = GlobalNpmPackageResolver::from_deno_dir( + let registry_url = NpmRegistryApi::default_url(); + let npm_cache = NpmCache::from_deno_dir( &dir, + cli_options.cache_setting(), + progress_bar.clone(), + ); + let api = NpmRegistryApi::new( + registry_url, + npm_cache.clone(), cli_options.reload_flag(), cli_options.cache_setting(), + progress_bar.clone(), + ); + let npm_resolver = NpmPackageResolver::new( + npm_cache.clone(), + api, cli_options.unstable() // don't do the unstable error when in the lsp || matches!(cli_options.sub_command(), DenoSubcommand::Lsp), cli_options.no_npm(), - progress_bar.clone(), ); let emit_options: deno_ast::EmitOptions = ts_config_result.ts_config.into(); @@ -253,6 +266,7 @@ impl ProcState { parsed_source_cache, maybe_resolver, maybe_file_watcher_reporter, + npm_cache, npm_resolver, cjs_resolutions: Default::default(), progress_bar, @@ -413,7 +427,6 @@ impl ProcState { .npm_resolver .add_package_reqs(npm_package_references) .await?; - self.npm_resolver.cache_packages().await?; self.prepare_node_std_graph().await?; } @@ -650,7 +663,6 @@ impl ProcState { } if !package_reqs.is_empty() { self.npm_resolver.add_package_reqs(package_reqs).await?; - self.npm_resolver.cache_packages().await?; } Ok(graph) diff --git a/cli/worker.rs b/cli/worker.rs index e766ec4aadc874..9b505e4f0c706a 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -356,7 +356,6 @@ pub async fn create_main_worker( ps.npm_resolver .add_package_reqs(vec![package_ref.req.clone()]) .await?; - ps.npm_resolver.cache_packages().await?; ps.prepare_node_std_graph().await?; let node_resolution = node::node_resolve_binary_export( &package_ref.req, diff --git a/ext/node/lib.rs b/ext/node/lib.rs index db4fe31786d486..da8ca300351348 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -30,7 +30,7 @@ pub trait NodePermissions { fn check_read(&mut self, path: &Path) -> Result<(), AnyError>; } -pub trait DenoDirNpmResolver { +pub trait RequireNpmResolver { fn resolve_package_folder_from_package( &self, specifier: &str, @@ -63,7 +63,7 @@ struct Unstable(pub bool); pub fn init( unstable: bool, - maybe_npm_resolver: Option>, + maybe_npm_resolver: Option>, ) -> Extension { Extension::builder() .js(include_js_files!( @@ -121,7 +121,7 @@ where P: NodePermissions + 'static, { let resolver = { - let resolver = state.borrow::>(); + let resolver = state.borrow::>(); resolver.clone() }; if resolver.ensure_read_permission(file_path).is_ok() { @@ -287,7 +287,7 @@ fn op_require_resolve_deno_dir( parent_filename: String, ) -> Option { check_unstable(state); - let resolver = state.borrow::>(); + let resolver = state.borrow::>(); resolver .resolve_package_folder_from_package( &request, @@ -300,7 +300,7 @@ fn op_require_resolve_deno_dir( #[op] fn op_require_is_deno_dir_package(state: &mut OpState, path: String) -> bool { check_unstable(state); - let resolver = state.borrow::>(); + let resolver = state.borrow::>(); resolver.in_npm_package(&PathBuf::from(path)) } @@ -470,7 +470,7 @@ fn op_require_try_self( return Ok(None); } - let resolver = state.borrow::>().clone(); + let resolver = state.borrow::>().clone(); let pkg = resolution::get_package_scope_config( &Url::from_file_path(parent_path.unwrap()).unwrap(), &*resolver, @@ -552,7 +552,7 @@ fn op_require_resolve_exports( parent_path: String, ) -> Result, AnyError> { check_unstable(state); - let resolver = state.borrow::>().clone(); + let resolver = state.borrow::>().clone(); let pkg_path = if resolver.in_npm_package(&PathBuf::from(&modules_path)) { modules_path @@ -594,7 +594,7 @@ where state, PathBuf::from(&filename).parent().unwrap(), )?; - let resolver = state.borrow::>().clone(); + let resolver = state.borrow::>().clone(); resolution::get_closest_package_json( &Url::from_file_path(filename).unwrap(), &*resolver, @@ -607,7 +607,7 @@ fn op_require_read_package_scope( package_json_path: String, ) -> Option { check_unstable(state); - let resolver = state.borrow::>().clone(); + let resolver = state.borrow::>().clone(); let package_json_path = PathBuf::from(package_json_path); PackageJson::load(&*resolver, package_json_path).ok() } @@ -624,7 +624,7 @@ where check_unstable(state); let parent_path = PathBuf::from(&parent_filename); ensure_read_permission::

(state, &parent_path)?; - let resolver = state.borrow::>().clone(); + let resolver = state.borrow::>().clone(); let pkg = PackageJson::load(&*resolver, parent_path.join("package.json"))?; if pkg.imports.is_some() { diff --git a/ext/node/package_json.rs b/ext/node/package_json.rs index ced64a1b40fdf5..95d773aeb3ad4b 100644 --- a/ext/node/package_json.rs +++ b/ext/node/package_json.rs @@ -2,7 +2,7 @@ use crate::NodeModuleKind; -use super::DenoDirNpmResolver; +use super::RequireNpmResolver; use deno_core::anyhow; use deno_core::anyhow::bail; use deno_core::error::AnyError; @@ -44,7 +44,7 @@ impl PackageJson { } pub fn load( - resolver: &dyn DenoDirNpmResolver, + resolver: &dyn RequireNpmResolver, path: PathBuf, ) -> Result { resolver.ensure_read_permission(&path)?; diff --git a/ext/node/resolution.rs b/ext/node/resolution.rs index a7428fe03a7f41..52ed06116e067c 100644 --- a/ext/node/resolution.rs +++ b/ext/node/resolution.rs @@ -15,7 +15,7 @@ use regex::Regex; use crate::errors; use crate::package_json::PackageJson; -use crate::DenoDirNpmResolver; +use crate::RequireNpmResolver; pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"]; pub static REQUIRE_CONDITIONS: &[&str] = &["require", "node"]; @@ -93,7 +93,7 @@ pub fn package_imports_resolve( referrer: &ModuleSpecifier, referrer_kind: NodeModuleKind, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { if name == "#" || name.starts_with("#/") || name.ends_with('/') { let reason = "is not a valid internal imports specifier name"; @@ -224,7 +224,7 @@ fn resolve_package_target_string( pattern: bool, internal: bool, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { if !subpath.is_empty() && !pattern && !target.ends_with('/') { return Err(throw_invalid_package_target( @@ -326,7 +326,7 @@ fn resolve_package_target( pattern: bool, internal: bool, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result, AnyError> { if let Some(target) = target.as_str() { return Ok(Some(resolve_package_target_string( @@ -441,7 +441,7 @@ pub fn package_exports_resolve( referrer: &ModuleSpecifier, referrer_kind: NodeModuleKind, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { if package_exports.contains_key(&package_subpath) && package_subpath.find('*').is_none() @@ -592,7 +592,7 @@ pub fn package_resolve( referrer: &ModuleSpecifier, referrer_kind: NodeModuleKind, conditions: &[&str], - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { let (package_name, package_subpath, _is_scoped) = parse_package_name(specifier, referrer)?; @@ -656,7 +656,7 @@ pub fn package_resolve( pub fn get_package_scope_config( referrer: &ModuleSpecifier, - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { let root_folder = npm_resolver .resolve_package_folder_from_path(&referrer.to_file_path().unwrap())?; @@ -666,7 +666,7 @@ pub fn get_package_scope_config( pub fn get_closest_package_json( url: &ModuleSpecifier, - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { let package_json_path = get_closest_package_json_path(url, npm_resolver)?; PackageJson::load(npm_resolver, package_json_path) @@ -674,7 +674,7 @@ pub fn get_closest_package_json( fn get_closest_package_json_path( url: &ModuleSpecifier, - npm_resolver: &dyn DenoDirNpmResolver, + npm_resolver: &dyn RequireNpmResolver, ) -> Result { let file_path = url.to_file_path().unwrap(); let mut current_dir = file_path.parent().unwrap(); diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index d4a96633d00121..306e1da5cfca7c 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -31,7 +31,7 @@ use deno_core::ModuleSpecifier; use deno_core::RuntimeOptions; use deno_core::SharedArrayBufferStore; use deno_core::SourceMapGetter; -use deno_node::DenoDirNpmResolver; +use deno_node::RequireNpmResolver; use deno_tls::rustls::RootCertStore; use deno_web::create_entangled_message_port; use deno_web::BlobStore; @@ -324,7 +324,7 @@ pub struct WebWorkerOptions { pub root_cert_store: Option, pub seed: Option, pub module_loader: Rc, - pub npm_resolver: Option>, + pub npm_resolver: Option>, pub create_web_worker_cb: Arc, pub preload_module_cb: Arc, pub pre_execute_module_cb: Arc, diff --git a/runtime/worker.rs b/runtime/worker.rs index 632805e6d47c1b..bce30b88eae875 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -25,7 +25,7 @@ use deno_core::ModuleSpecifier; use deno_core::RuntimeOptions; use deno_core::SharedArrayBufferStore; use deno_core::SourceMapGetter; -use deno_node::DenoDirNpmResolver; +use deno_node::RequireNpmResolver; use deno_tls::rustls::RootCertStore; use deno_web::BlobStore; use log::debug; @@ -75,7 +75,7 @@ pub struct WorkerOptions { pub root_cert_store: Option, pub seed: Option, pub module_loader: Rc, - pub npm_resolver: Option>, + pub npm_resolver: Option>, // Callbacks invoked when creating new instance of WebWorker pub create_web_worker_cb: Arc, pub web_worker_preload_module_cb: Arc,