diff --git a/workspaces/arborist/lib/arborist/build-ideal-tree.js b/workspaces/arborist/lib/arborist/build-ideal-tree.js index 0d936d8ef7768..d04ddf27b6965 100644 --- a/workspaces/arborist/lib/arborist/build-ideal-tree.js +++ b/workspaces/arborist/lib/arborist/build-ideal-tree.js @@ -31,75 +31,75 @@ const Link = require('../link.js') const addRmPkgDeps = require('../add-rm-pkg-deps.js') const optionalSet = require('../optional-set.js') const { checkEngine, checkPlatform } = require('npm-install-checks') - const relpath = require('../relpath.js') +const resetDepFlags = require('../reset-dep-flags.js') // note: some of these symbols are shared so we can hit // them with unit tests and reuse them across mixins -const _complete = Symbol('complete') -const _depsSeen = Symbol('depsSeen') -const _depsQueue = Symbol('depsQueue') -const _currentDep = Symbol('currentDep') const _updateAll = Symbol.for('updateAll') -const _mutateTree = Symbol('mutateTree') const _flagsSuspect = Symbol.for('flagsSuspect') const _workspaces = Symbol.for('workspaces') -const _prune = Symbol('prune') -const _preferDedupe = Symbol('preferDedupe') -const _parseSettings = Symbol('parseSettings') -const _initTree = Symbol('initTree') -const _applyUserRequests = Symbol('applyUserRequests') -const _applyUserRequestsToNode = Symbol('applyUserRequestsToNode') -const _inflateAncientLockfile = Symbol('inflateAncientLockfile') -const _buildDeps = Symbol('buildDeps') -const _buildDepStep = Symbol('buildDepStep') -const _nodeFromEdge = Symbol('nodeFromEdge') -const _nodeFromSpec = Symbol('nodeFromSpec') -const _fetchManifest = Symbol('fetchManifest') -const _problemEdges = Symbol('problemEdges') -const _manifests = Symbol('manifests') -const _loadWorkspaces = Symbol.for('loadWorkspaces') -const _linkFromSpec = Symbol('linkFromSpec') -const _loadPeerSet = Symbol('loadPeerSet') +const _setWorkspaces = Symbol.for('setWorkspaces') const _updateNames = Symbol.for('updateNames') -const _fixDepFlags = Symbol('fixDepFlags') -const _resolveLinks = Symbol('resolveLinks') -const _rootNodeFromPackage = Symbol('rootNodeFromPackage') -const _add = Symbol('add') const _resolvedAdd = Symbol.for('resolvedAdd') -const _queueNamedUpdates = Symbol('queueNamedUpdates') -const _queueVulnDependents = Symbol('queueVulnDependents') -const _avoidRange = Symbol('avoidRange') -const _shouldUpdateNode = Symbol('shouldUpdateNode') -const resetDepFlags = require('../reset-dep-flags.js') -const _loadFailures = Symbol('loadFailures') -const _pruneFailedOptional = Symbol('pruneFailedOptional') -const _linkNodes = Symbol('linkNodes') -const _follow = Symbol('follow') -const _installStrategy = Symbol('installStrategy') -const _globalRootNode = Symbol('globalRootNode') const _usePackageLock = Symbol.for('usePackageLock') const _rpcache = Symbol.for('realpathCache') const _stcache = Symbol.for('statCache') -const _strictPeerDeps = Symbol('strictPeerDeps') -const _checkEngineAndPlatform = Symbol('checkEngineAndPlatform') -const _virtualRoots = Symbol('virtualRoots') -const _virtualRoot = Symbol('virtualRoot') const _includeWorkspaceRoot = Symbol.for('includeWorkspaceRoot') -const _failPeerConflict = Symbol('failPeerConflict') -const _explainPeerConflict = Symbol('explainPeerConflict') -const _edgesOverridden = Symbol('edgesOverridden') // exposed symbol for unit testing the placeDep method directly const _peerSetSource = Symbol.for('peerSetSource') // used by Reify mixin const _force = Symbol.for('force') -const _explicitRequests = Symbol('explicitRequests') const _global = Symbol.for('global') const _idealTreePrune = Symbol.for('idealTreePrune') +// Push items in, pop them sorted by depth and then path +class DepsQueue { + #deps = [] + #sorted = true + + get length () { + return this.#deps.length + } + + push (item) { + if (!this.#deps.includes(item)) { + this.#sorted = false + this.#deps.push(item) + } + } + + pop () { + if (!this.#sorted) { + // sort physically shallower deps up to the front of the queue, because + // they'll affect things deeper in, then alphabetical + this.#deps.sort((a, b) => + (a.depth - b.depth) || localeCompare(a.path, b.path)) + this.#sorted = true + } + return this.#deps.shift() + } +} + module.exports = cls => class IdealTreeBuilder extends cls { + #complete + #currentDep = null + #depsQueue = new DepsQueue() + #depsSeen = new Set() + #explicitRequests = new Set() + #follow + #installStrategy + #linkNodes = new Set() + #loadFailures = new Set() + #manifests = new Map() + #mutateTree = false + #preferDedupe = false + #prune + #strictPeerDeps + #virtualRoots = new Map() + constructor (options) { super(options) @@ -123,7 +123,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { this[_workspaces] = workspaces || [] this[_force] = !!force - this[_strictPeerDeps] = !!strictPeerDeps + this.#strictPeerDeps = !!strictPeerDeps this.idealTree = idealTree this.installLinks = installLinks @@ -131,38 +131,27 @@ module.exports = cls => class IdealTreeBuilder extends cls { this[_usePackageLock] = packageLock this[_global] = !!global - this[_installStrategy] = global ? 'shallow' : installStrategy - this[_follow] = !!follow + this.#installStrategy = global ? 'shallow' : installStrategy + this.#follow = !!follow if (this[_workspaces].length && this[_global]) { throw new Error('Cannot operate on workspaces in global mode') } - this[_explicitRequests] = new Set() - this[_preferDedupe] = false - this[_depsSeen] = new Set() - this[_depsQueue] = [] - this[_currentDep] = null - this[_updateNames] = [] this[_updateAll] = false - this[_mutateTree] = false - this[_loadFailures] = new Set() - this[_linkNodes] = new Set() - this[_manifests] = new Map() - this[_edgesOverridden] = new Set() + this[_updateNames] = [] this[_resolvedAdd] = [] // a map of each module in a peer set to the thing that depended on // that set of peers in the first place. Use a WeakMap so that we // don't hold onto references for nodes that are garbage collected. this[_peerSetSource] = new WeakMap() - this[_virtualRoots] = new Map() this[_includeWorkspaceRoot] = includeWorkspaceRoot } get explicitRequests () { - return new Set(this[_explicitRequests]) + return new Set(this.#explicitRequests) } // public method @@ -195,19 +184,19 @@ module.exports = cls => class IdealTreeBuilder extends cls { // from there, we start adding nodes to it to satisfy the deps requested // by the package.json in the root. - this[_parseSettings](options) + this.#parseSettings(options) // start tracker block this.addTracker('idealTree') try { - await this[_initTree]() - await this[_inflateAncientLockfile]() - await this[_applyUserRequests](options) - await this[_buildDeps]() - await this[_fixDepFlags]() - await this[_pruneFailedOptional]() - await this[_checkEngineAndPlatform]() + await this.#initTree() + await this.#inflateAncientLockfile() + await this.#applyUserRequests(options) + await this.#buildDeps() + await this.#fixDepFlags() + await this.#pruneFailedOptional() + await this.#checkEngineAndPlatform() } finally { process.emit('timeEnd', 'idealTree') this.finishTracker('idealTree') @@ -216,7 +205,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { return treeCheck(this.idealTree) } - async [_checkEngineAndPlatform] () { + async #checkEngineAndPlatform () { const { engineStrict, npmVersion, nodeVersion } = this.options for (const node of this.idealTree.inventory.values()) { if (!node.optional) { @@ -237,7 +226,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { } } - [_parseSettings] (options) { + #parseSettings (options) { const update = options.update === true ? { all: true } : Array.isArray(options.update) ? { names: options.update } : options.update || {} @@ -246,8 +235,8 @@ module.exports = cls => class IdealTreeBuilder extends cls { update.names = [] } - this[_complete] = !!options.complete - this[_preferDedupe] = !!options.preferDedupe + this.#complete = !!options.complete + this.#preferDedupe = !!options.preferDedupe // validates list of update names, they must // be dep names only, no semver ranges are supported @@ -267,11 +256,11 @@ module.exports = cls => class IdealTreeBuilder extends cls { this[_updateAll] = update.all // we prune by default unless explicitly set to boolean false - this[_prune] = options.prune !== false + this.#prune = options.prune !== false // set if we add anything, but also set here if we know we'll make // changes and thus have to maybe prune later. - this[_mutateTree] = !!( + this.#mutateTree = !!( options.add || options.rm || update.all || @@ -281,20 +270,23 @@ module.exports = cls => class IdealTreeBuilder extends cls { // load the initial tree, either the virtualTree from a shrinkwrap, // or just the root node from a package.json - [_initTree] () { + async #initTree () { process.emit('time', 'idealTree:init') - return ( - this[_global] ? this[_globalRootNode]() - : rpj(this.path + '/package.json').then( - pkg => this[_rootNodeFromPackage](pkg), - er => { - if (er.code === 'EJSONPARSE') { - throw er - } - return this[_rootNodeFromPackage]({}) + let root + if (this[_global]) { + root = await this.#globalRootNode() + } else { + try { + const pkg = await rpj(this.path + '/package.json') + root = await this.#rootNodeFromPackage(pkg) + } catch (err) { + if (err.code === 'EJSONPARSE') { + throw err } - )) - .then(root => this[_loadWorkspaces](root)) + root = await this.#rootNodeFromPackage({}) + } + } + return this[_setWorkspaces](root) // ok to not have a virtual tree. probably initial install. // When updating all, we load the shrinkwrap, but don't bother // to build out the full virtual tree from it, since we'll be @@ -336,12 +328,18 @@ module.exports = cls => class IdealTreeBuilder extends cls { // the depsQueue so that we'll fix it later depth({ tree, - getChildren: (node) => [...node.edgesOut.values()].map(edge => edge.to), + getChildren: (node) => { + const children = [] + for (const edge of node.edgesOut.values()) { + children.push(edge.to) + } + return children + }, filter: node => node, visit: node => { for (const edge of node.edgesOut.values()) { if (!edge.valid) { - this[_depsQueue].push(node) + this.#depsQueue.push(node) break // no need to continue the loop after the first hit } } @@ -356,8 +354,8 @@ module.exports = cls => class IdealTreeBuilder extends cls { }) } - async [_globalRootNode] () { - const root = await this[_rootNodeFromPackage]({ dependencies: {} }) + async #globalRootNode () { + const root = await this.#rootNodeFromPackage({ dependencies: {} }) // this is a gross kludge to handle the fact that we don't save // metadata on the root node in global installs, because the "root" // node is something like /usr/local/lib. @@ -371,7 +369,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { return root } - async [_rootNodeFromPackage] (pkg) { + async #rootNodeFromPackage (pkg) { // if the path doesn't exist, then we explode at this point. Note that // this is not a problem for reify(), since it creates the root path // before ever loading trees. @@ -414,19 +412,19 @@ module.exports = cls => class IdealTreeBuilder extends cls { // process the add/rm requests by modifying the root node, and the // update.names request by queueing nodes dependent on those named. - async [_applyUserRequests] (options) { + async #applyUserRequests (options) { process.emit('time', 'idealTree:userRequests') const tree = this.idealTree.target if (!this[_workspaces].length) { - await this[_applyUserRequestsToNode](tree, options) + await this.#applyUserRequestsToNode(tree, options) } else { const nodes = this.workspaceNodes(tree, this[_workspaces]) if (this[_includeWorkspaceRoot]) { nodes.push(tree) } const appliedRequests = nodes.map( - node => this[_applyUserRequestsToNode](node, options) + node => this.#applyUserRequestsToNode(node, options) ) await Promise.all(appliedRequests) } @@ -434,12 +432,12 @@ module.exports = cls => class IdealTreeBuilder extends cls { process.emit('timeEnd', 'idealTree:userRequests') } - async [_applyUserRequestsToNode] (tree, options) { + async #applyUserRequestsToNode (tree, options) { // If we have a list of package names to update, and we know it's // going to update them wherever they are, add any paths into those // named nodes to the buildIdealTree queue. if (!this[_global] && this[_updateNames].length) { - this[_queueNamedUpdates]() + this.#queueNamedUpdates() } // global updates only update the globalTop nodes, but we need to know @@ -448,7 +446,8 @@ module.exports = cls => class IdealTreeBuilder extends cls { if (this[_global] && (this[_updateAll] || this[_updateNames].length)) { const nm = resolve(this.path, 'node_modules') const paths = await readdirScoped(nm).catch(() => []) - for (const name of paths.map((p) => p.replace(/\\/g, '/'))) { + for (const p of paths) { + const name = p.replace(/\\/g, '/') tree.package.dependencies = tree.package.dependencies || {} const updateName = this[_updateNames].includes(name) if (this[_updateAll] || updateName) { @@ -470,7 +469,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { } if (this.auditReport && this.auditReport.size > 0) { - await this[_queueVulnDependents](options) + await this.#queueVulnDependents(options) } const { add, rm } = options @@ -478,12 +477,12 @@ module.exports = cls => class IdealTreeBuilder extends cls { if (rm && rm.length) { addRmPkgDeps.rm(tree.package, rm) for (const name of rm) { - this[_explicitRequests].add({ from: tree, name, action: 'DELETE' }) + this.#explicitRequests.add({ from: tree, name, action: 'DELETE' }) } } if (add && add.length) { - await this[_add](tree, options) + await this.#add(tree, options) } // triggers a refresh of all edgesOut. this has to be done BEFORE @@ -495,19 +494,19 @@ module.exports = cls => class IdealTreeBuilder extends cls { for (const spec of this[_resolvedAdd]) { if (spec.tree === tree) { - this[_explicitRequests].add(tree.edgesOut.get(spec.name)) + this.#explicitRequests.add(tree.edgesOut.get(spec.name)) } } for (const name of globalExplicitUpdateNames) { - this[_explicitRequests].add(tree.edgesOut.get(name)) + this.#explicitRequests.add(tree.edgesOut.get(name)) } - this[_depsQueue].push(tree) + this.#depsQueue.push(tree) } // This returns a promise because we might not have the name yet, and need to // call pacote.manifest to find the name. - async [_add] (tree, { add, saveType = null, saveBundle = false }) { + async #add (tree, { add, saveType = null, saveBundle = false }) { // If we have a link it will need to be added relative to the target's path const path = tree.target.path @@ -564,7 +563,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { // what's in the bundle at each published manifest. Without that, we // can't possibly fix bundled deps without breaking a ton of other stuff, // and leaving the user subject to getting it overwritten later anyway. - async [_queueVulnDependents] (options) { + async #queueVulnDependents (options) { for (const vuln of this.auditReport.values()) { for (const node of vuln.nodes) { const bundler = node.getBundler() @@ -584,7 +583,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { for (const edge of node.edgesIn) { this.addTracker('idealTree', edge.from.name, edge.from.location) - this[_depsQueue].push(edge.from) + this.#depsQueue.push(edge.from) } } } @@ -629,7 +628,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { log.warn('audit', `Updating ${fixName} to ${version}, ` + `which is ${breakingMessage}.`) - await this[_add](node, { add: [`${fixName}@${version}`] }) + await this.#add(node, { add: [`${fixName}@${version}`] }) nodesTouched.add(node) } } @@ -639,7 +638,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { } } - [_avoidRange] (name) { + #avoidRange (name) { if (!this.auditReport) { return null } @@ -650,7 +649,7 @@ module.exports = cls => class IdealTreeBuilder extends cls { return vuln.range } - [_queueNamedUpdates] () { + #queueNamedUpdates () { // ignore top nodes, since they are not loaded the same way, and // probably have their own project associated with them. @@ -661,25 +660,19 @@ module.exports = cls => class IdealTreeBuilder extends cls { // XXX this could be faster by doing a series of inventory.query('name') // calls rather than walking over everything in the tree. - const set = this.idealTree.inventory - .filter(n => this[_shouldUpdateNode](n)) - // XXX add any invalid edgesOut to the queue - for (const node of set) { - for (const edge of node.edgesIn) { - this.addTracker('idealTree', edge.from.name, edge.from.location) - this[_depsQueue].push(edge.from) + for (const node of this.idealTree.inventory.values()) { + // XXX add any invalid edgesOut to the queue + if (this[_updateNames].includes(node.name) && + !node.isTop && !node.inDepBundle && !node.inShrinkwrap) { + for (const edge of node.edgesIn) { + this.addTracker('idealTree', edge.from.name, edge.from.location) + this.#depsQueue.push(edge.from) + } } } } - [_shouldUpdateNode] (node) { - return this[_updateNames].includes(node.name) && - !node.isTop && - !node.inDepBundle && - !node.inShrinkwrap - } - - async [_inflateAncientLockfile] () { + async #inflateAncientLockfile () { const { meta, inventory } = this.idealTree const ancient = meta.ancientLockfile const old = meta.loadedFromDisk && !(meta.originalLockfileVersion >= 2) @@ -764,38 +757,33 @@ This is a one-time fix-up, please be patient... // at this point we have a virtual tree with the actual root node's // package deps, which may be partly or entirely incomplete, invalid // or extraneous. - [_buildDeps] () { + #buildDeps () { process.emit('time', 'idealTree:buildDeps') const tree = this.idealTree.target tree.assertRootOverrides() - this[_depsQueue].push(tree) + this.#depsQueue.push(tree) // XXX also push anything that depends on a node with a name // in the override list log.silly('idealTree', 'buildDeps') this.addTracker('idealTree', tree.name, '') - return this[_buildDepStep]() + return this.#buildDepStep() .then(() => process.emit('timeEnd', 'idealTree:buildDeps')) } - async [_buildDepStep] () { + async #buildDepStep () { // removes tracker of previous dependency in the queue - if (this[_currentDep]) { - const { location, name } = this[_currentDep] + if (this.#currentDep) { + const { location, name } = this.#currentDep process.emit('timeEnd', `idealTree:${location || '#root'}`) this.finishTracker('idealTree', name, location) - this[_currentDep] = null + this.#currentDep = null } - if (!this[_depsQueue].length) { - return this[_resolveLinks]() + if (!this.#depsQueue.length) { + return this.#resolveLinks() } - // sort physically shallower deps up to the front of the queue, - // because they'll affect things deeper in, then alphabetical - this[_depsQueue].sort((a, b) => - (a.depth - b.depth) || localeCompare(a.path, b.path)) - - const node = this[_depsQueue].shift() + const node = this.#depsQueue.pop() const bd = node.package.bundleDependencies const hasBundle = bd && Array.isArray(bd) && bd.length const { hasShrinkwrap } = node @@ -804,14 +792,14 @@ This is a one-time fix-up, please be patient... // tree, skip over it and process the rest of the queue. If a node has // a shrinkwrap, also skip it, because it's going to get its deps // satisfied by whatever's in that file anyway. - if (this[_depsSeen].has(node) || + if (this.#depsSeen.has(node) || node.root !== this.idealTree || - hasShrinkwrap && !this[_complete]) { - return this[_buildDepStep]() + hasShrinkwrap && !this.#complete) { + return this.#buildDepStep() } - this[_depsSeen].add(node) - this[_currentDep] = node + this.#depsSeen.add(node) + this.#currentDep = node process.emit('time', `idealTree:${node.location || '#root'}`) // if we're loading a _complete_ ideal tree, for a --package-lock-only @@ -821,7 +809,7 @@ This is a one-time fix-up, please be patient... // ideal tree by reading bundles/shrinkwraps in place. // Don't bother if the node is from the actual tree and hasn't // been resolved, because we can't fetch it anyway, could be anything! - const crackOpen = this[_complete] && + const crackOpen = this.#complete && node !== this.idealTree && node.resolved && (hasBundle || hasShrinkwrap) @@ -891,7 +879,7 @@ This is a one-time fix-up, please be patient... const tasks = [] const peerSource = this[_peerSetSource].get(node) || node - for (const edge of this[_problemEdges](node)) { + for (const edge of this.#problemEdges(node)) { if (edge.peerConflicted) { continue } @@ -902,7 +890,7 @@ This is a one-time fix-up, please be patient... // so we VR the node itself if the edge is not a peer const source = edge.peer ? peerSource : node - const virtualRoot = this[_virtualRoot](source, true) + const virtualRoot = this.#virtualRoot(source, true) // reuse virtual root if we already have one, but don't // try to do the override ahead of time, since we MAY be able // to create a more correct tree than the virtual root could. @@ -926,7 +914,7 @@ This is a one-time fix-up, please be patient... const required = new Set([edge.from]) const parent = edge.peer ? virtualRoot : null const dep = vrDep && vrDep.satisfies(edge) ? vrDep - : await this[_nodeFromEdge](edge, parent, null, required) + : await this.#nodeFromEdge(edge, parent, null, required) /* istanbul ignore next */ debug(() => { @@ -938,25 +926,24 @@ This is a one-time fix-up, please be patient... tasks.push({ edge, dep }) } - const placeDeps = tasks - .sort((a, b) => localeCompare(a.edge.name, b.edge.name)) - .map(({ edge, dep }) => new PlaceDep({ + const placeDeps = tasks.sort((a, b) => localeCompare(a.edge.name, b.edge.name)) + + const promises = [] + for (const { edge, dep } of placeDeps) { + const pd = new PlaceDep({ edge, dep, auditReport: this.auditReport, - explicitRequest: this[_explicitRequests].has(edge), + explicitRequest: this.#explicitRequests.has(edge), force: this[_force], installLinks: this.installLinks, - installStrategy: this[_installStrategy], + installStrategy: this.#installStrategy, legacyPeerDeps: this.legacyPeerDeps, - preferDedupe: this[_preferDedupe], - strictPeerDeps: this[_strictPeerDeps], + preferDedupe: this.#preferDedupe, + strictPeerDeps: this.#strictPeerDeps, updateNames: this[_updateNames], - })) - - const promises = [] - for (const pd of placeDeps) { + }) // placing a dep is actually a tree of placing the dep itself // and all of its peer group that aren't already met by the tree depth({ @@ -971,18 +958,18 @@ This is a one-time fix-up, please be patient... // we placed something, that means we changed the tree if (placed.errors.length) { - this[_loadFailures].add(placed) + this.#loadFailures.add(placed) } - this[_mutateTree] = true + this.#mutateTree = true if (cpd.canPlaceSelf === OK) { for (const edgeIn of placed.edgesIn) { if (edgeIn === edge) { continue } const { from, valid, peerConflicted } = edgeIn - if (!peerConflicted && !valid && !this[_depsSeen].has(from)) { + if (!peerConflicted && !valid && !this.#depsSeen.has(from)) { this.addTracker('idealTree', from.name, from.location) - this[_depsQueue].push(edgeIn.from) + this.#depsQueue.push(edgeIn.from) } } } else { @@ -1000,8 +987,8 @@ This is a one-time fix-up, please be patient... if (!valid && !peerConflicted) { // if it's already been visited, we have to re-visit // otherwise, just enqueue normally. - this[_depsSeen].delete(edgeIn.from) - this[_depsQueue].push(edgeIn.from) + this.#depsSeen.delete(edgeIn.from) + this.#depsQueue.push(edgeIn.from) } } } @@ -1018,45 +1005,48 @@ This is a one-time fix-up, please be patient... // lastly, also check for the missing deps of the node we placed, // and any holes created by pruning out conflicted peer sets. - this[_depsQueue].push(placed) + this.#depsQueue.push(placed) for (const dep of pd.needEvaluation) { - this[_depsSeen].delete(dep) - this[_depsQueue].push(dep) + this.#depsSeen.delete(dep) + this.#depsQueue.push(dep) } // pre-fetch any problem edges, since we'll need these soon // if it fails at this point, though, dont' worry because it // may well be an optional dep that has gone missing. it'll // fail later anyway. - promises.push(...this[_problemEdges](placed).map(e => - this[_fetchManifest](npa.resolve(e.name, e.spec, fromPath(placed, e))) - .catch(er => null))) + for (const e of this.#problemEdges(placed)) { + promises.push( + this.#fetchManifest(npa.resolve(e.name, e.spec, fromPath(placed, e))) + .catch(er => null) + ) + } }, }) } for (const { to } of node.edgesOut.values()) { if (to && to.isLink && to.target) { - this[_linkNodes].add(to) + this.#linkNodes.add(to) } } await Promise.all(promises) - return this[_buildDepStep]() + return this.#buildDepStep() } // loads a node from an edge, and then loads its peer deps (and their // peer deps, on down the line) into a virtual root parent. - async [_nodeFromEdge] (edge, parent_, secondEdge, required) { + async #nodeFromEdge (edge, parent_, secondEdge, required) { // create a virtual root node with the same deps as the node that // is requesting this one, so that we can get all the peer deps in // a context where they're likely to be resolvable. // Note that the virtual root will also have virtual copies of the // targets of any child Links, so that they resolve appropriately. - const parent = parent_ || this[_virtualRoot](edge.from) + const parent = parent_ || this.#virtualRoot(edge.from) const spec = npa.resolve(edge.name, edge.spec, edge.from.path) - const first = await this[_nodeFromSpec](edge.name, spec, parent, edge) + const first = await this.#nodeFromSpec(edge.name, spec, parent, edge) // we might have a case where the parent has a peer dependency on // `foo@*` which resolves to v2, but another dep in the set has a @@ -1071,7 +1061,7 @@ This is a one-time fix-up, please be patient... secondEdge.from.path ) const second = secondEdge && !secondEdge.valid - ? await this[_nodeFromSpec](edge.name, spec2, parent, secondEdge) + ? await this.#nodeFromSpec(edge.name, spec2, parent, secondEdge) : null // pick the second one if they're both happy with that, otherwise first @@ -1098,12 +1088,12 @@ This is a one-time fix-up, please be patient... } // otherwise, we have to make sure that our peers can go along with us. - return this[_loadPeerSet](node, required) + return this.#loadPeerSet(node, required) } - [_virtualRoot] (node, reuse = false) { - if (reuse && this[_virtualRoots].has(node)) { - return this[_virtualRoots].get(node) + #virtualRoot (node, reuse = false) { + if (reuse && this.#virtualRoots.has(node)) { + return this.#virtualRoots.get(node) } const vr = new Node({ @@ -1126,11 +1116,11 @@ This is a one-time fix-up, please be patient... } } - this[_virtualRoots].set(node, vr) + this.#virtualRoots.set(node, vr) return vr } - [_problemEdges] (node) { + #problemEdges (node) { // skip over any bundled deps, they're not our problem. // Note that this WILL fetch bundled meta-deps which are also dependencies // but not listed as bundled deps. When reifying, we first unpack any @@ -1145,85 +1135,90 @@ This is a one-time fix-up, please be patient... : node.package.bundleDependencies const bundled = new Set(bd || []) - return [...node.edgesOut.values()] - .filter(edge => { - // If it's included in a bundle, we take whatever is specified. - if (bundled.has(edge.name)) { - return false - } - - // If it's already been logged as a load failure, skip it. - if (edge.to && this[_loadFailures].has(edge.to)) { - return false - } + const problems = [] + for (const edge of node.edgesOut.values()) { + // If it's included in a bundle, we take whatever is specified. + if (bundled.has(edge.name)) { + continue + } - // If it's shrinkwrapped, we use what the shrinkwap wants. - if (edge.to && edge.to.inShrinkwrap) { - return false - } + // If it's already been logged as a load failure, skip it. + if (edge.to && this.#loadFailures.has(edge.to)) { + continue + } - // If the edge has no destination, that's a problem, unless - // if it's peerOptional and not explicitly requested. - if (!edge.to) { - return edge.type !== 'peerOptional' || - this[_explicitRequests].has(edge) - } + // If it's shrinkwrapped, we use what the shrinkwap wants. + if (edge.to && edge.to.inShrinkwrap) { + continue + } - // If the edge has an error, there's a problem. - if (!edge.valid) { - return true + // If the edge has no destination, that's a problem, unless + // if it's peerOptional and not explicitly requested. + if (!edge.to) { + if (edge.type !== 'peerOptional' || + this.#explicitRequests.has(edge)) { + problems.push(edge) } + continue + } - // If the edge is a workspace, and it's valid, leave it alone - if (edge.to.isWorkspace) { - return false - } + // If the edge has an error, there's a problem. + if (!edge.valid) { + problems.push(edge) + continue + } - // user explicitly asked to update this package by name, problem - if (this[_updateNames].includes(edge.name)) { - return true - } + // If the edge is a workspace, and it's valid, leave it alone + if (edge.to.isWorkspace) { + continue + } - // fixing a security vulnerability with this package, problem - if (this.auditReport && this.auditReport.isVulnerable(edge.to)) { - return true - } + // user explicitly asked to update this package by name, problem + if (this[_updateNames].includes(edge.name)) { + problems.push(edge) + continue + } - // user has explicitly asked to install this package, problem - if (this[_explicitRequests].has(edge)) { - return true - } + // fixing a security vulnerability with this package, problem + if (this.auditReport && this.auditReport.isVulnerable(edge.to)) { + problems.push(edge) + continue + } - // No problems! - return false - }) + // user has explicitly asked to install this package, problem + if (this.#explicitRequests.has(edge)) { + problems.push(edge) + continue + } + } + return problems } - async [_fetchManifest] (spec) { + async #fetchManifest (spec) { const options = { ...this.options, - avoid: this[_avoidRange](spec.name), + avoid: this.#avoidRange(spec.name), } // get the intended spec and stored metadata from yarn.lock file, // if available and valid. spec = this.idealTree.meta.checkYarnLock(spec, options) - if (this[_manifests].has(spec.raw)) { - return this[_manifests].get(spec.raw) + if (this.#manifests.has(spec.raw)) { + return this.#manifests.get(spec.raw) } else { const cleanRawSpec = cleanUrl(spec.rawSpec) log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, cleanRawSpec)) const p = pacote.manifest(spec, options) .then(mani => { - this[_manifests].set(spec.raw, mani) + this.#manifests.set(spec.raw, mani) return mani }) - this[_manifests].set(spec.raw, p) + this.#manifests.set(spec.raw, p) return p } } - [_nodeFromSpec] (name, spec, parent, edge) { + #nodeFromSpec (name, spec, parent, edge) { // pacote will slap integrity on its options, so we have to clone // the object so it doesn't get mutated. // Don't bother to load the manifest for link deps, because the target @@ -1234,7 +1229,7 @@ This is a one-time fix-up, please be patient... // spec is a directory, link it unless installLinks is set or it's a workspace // TODO post arborist refactor, will need to check for installStrategy=linked if (spec.type === 'directory' && (isWorkspace || !installLinks)) { - return this[_linkFromSpec](name, spec, parent, edge) + return this.#linkFromSpec(name, spec, parent, edge) } // if the spec matches a workspace name, then see if the workspace node will @@ -1249,7 +1244,7 @@ This is a one-time fix-up, please be patient... // spec isn't a directory, and either isn't a workspace or the workspace we have // doesn't satisfy the edge. try to fetch a manifest and build a node from that. - return this[_fetchManifest](spec) + return this.#fetchManifest(spec) .then(pkg => new Node({ name, pkg, parent, installLinks, legacyPeerDeps }), error => { error.requiredBy = edge.from.location || '.' @@ -1263,17 +1258,17 @@ This is a one-time fix-up, please be patient... installLinks, legacyPeerDeps, }) - this[_loadFailures].add(n) + this.#loadFailures.add(n) return n }) } - [_linkFromSpec] (name, spec, parent, edge) { + #linkFromSpec (name, spec, parent, edge) { const realpath = spec.fetchSpec const { installLinks, legacyPeerDeps } = this return rpj(realpath + '/package.json').catch(() => ({})).then(pkg => { const link = new Link({ name, parent, realpath, pkg, installLinks, legacyPeerDeps }) - this[_linkNodes].add(link) + this.#linkNodes.add(link) return link }) } @@ -1291,7 +1286,7 @@ This is a one-time fix-up, please be patient... // gets placed first. In non-strict mode, we behave strictly if the // virtual root is based on the root project, and allow non-peer parent // deps to override, but throw if no preference can be determined. - async [_loadPeerSet] (node, required) { + async #loadPeerSet (node, required) { const peerEdges = [...node.edgesOut.values()] // we typically only install non-optional peers, but we have to // factor them into the peerSet so that we can avoid conflicts @@ -1307,12 +1302,12 @@ This is a one-time fix-up, please be patient... const parentEdge = node.parent.edgesOut.get(edge.name) const { isProjectRoot, isWorkspace } = node.parent.sourceReference const isMine = isProjectRoot || isWorkspace - const conflictOK = this[_force] || !isMine && !this[_strictPeerDeps] + const conflictOK = this[_force] || !isMine && !this.#strictPeerDeps if (!edge.to) { if (!parentEdge) { // easy, just put the thing there - await this[_nodeFromEdge](edge, node.parent, null, required) + await this.#nodeFromEdge(edge, node.parent, null, required) continue } else { // if the parent's edge is very broad like >=1, and the edge in @@ -1323,7 +1318,7 @@ This is a one-time fix-up, please be patient... // a conflict. this is always a problem in strict mode, never // in force mode, and a problem in non-strict mode if this isn't // on behalf of our project. in all such cases, we warn at least. - const dep = await this[_nodeFromEdge]( + const dep = await this.#nodeFromEdge( parentEdge, node.parent, edge, @@ -1344,7 +1339,7 @@ This is a one-time fix-up, please be patient... } // problem - this[_failPeerConflict](edge, parentEdge) + this.#failPeerConflict(edge, parentEdge) } } @@ -1352,9 +1347,9 @@ This is a one-time fix-up, please be patient... // See if the thing we WOULD be happy with is also going to satisfy // the other dependents on the current node. const current = edge.to - const dep = await this[_nodeFromEdge](edge, null, null, required) + const dep = await this.#nodeFromEdge(edge, null, null, required) if (dep.canReplace(current)) { - await this[_nodeFromEdge](edge, node.parent, null, required) + await this.#nodeFromEdge(edge, node.parent, null, required) continue } @@ -1371,17 +1366,17 @@ This is a one-time fix-up, please be patient... } // ok, it's the root, or we're in unforced strict mode, so this is bad - this[_failPeerConflict](edge, parentEdge) + this.#failPeerConflict(edge, parentEdge) } return node } - [_failPeerConflict] (edge, currentEdge) { - const expl = this[_explainPeerConflict](edge, currentEdge) + #failPeerConflict (edge, currentEdge) { + const expl = this.#explainPeerConflict(edge, currentEdge) throw Object.assign(new Error('unable to resolve dependency tree'), expl) } - [_explainPeerConflict] (edge, currentEdge) { + #explainPeerConflict (edge, currentEdge) { const node = edge.from const curNode = node.resolve(edge.name) const current = curNode.explain() @@ -1393,12 +1388,12 @@ This is a one-time fix-up, please be patient... // the tree handling logic. currentEdge: currentEdge ? currentEdge.explain() : null, edge: edge.explain(), - strictPeerDeps: this[_strictPeerDeps], + strictPeerDeps: this.#strictPeerDeps, force: this[_force], } } - // go through all the links in the this[_linkNodes] set + // go through all the links in the this.#linkNodes set // for each one: // - if outside the root, ignore it, assume it's fine, it's not our problem // - if a node in the tree already, assign the target to that node. @@ -1406,9 +1401,9 @@ This is a one-time fix-up, please be patient... // and add it to the _depsQueue // // call buildDepStep if anything was added to the queue, otherwise we're done - [_resolveLinks] () { - for (const link of this[_linkNodes]) { - this[_linkNodes].delete(link) + #resolveLinks () { + for (const link of this.#linkNodes) { + this.#linkNodes.delete(link) // link we never ended up placing, skip it if (link.root !== this.idealTree) { @@ -1419,34 +1414,34 @@ This is a one-time fix-up, please be patient... const external = !link.target.isDescendantOf(tree) // outside the root, somebody else's problem, ignore it - if (external && !this[_follow]) { + if (external && !this.#follow) { continue } // didn't find a parent for it or it has not been seen yet // so go ahead and process it. const unseenLink = (link.target.parent || link.target.fsParent) && - !this[_depsSeen].has(link.target) + !this.#depsSeen.has(link.target) - if (this[_follow] && + if (this.#follow && !link.target.parent && !link.target.fsParent || unseenLink) { this.addTracker('idealTree', link.target.name, link.target.location) - this[_depsQueue].push(link.target) + this.#depsQueue.push(link.target) } } - if (this[_depsQueue].length) { - return this[_buildDepStep]() + if (this.#depsQueue.length) { + return this.#buildDepStep() } } - [_fixDepFlags] () { + #fixDepFlags () { process.emit('time', 'idealTree:fixDepFlags') const metaFromDisk = this.idealTree.meta.loadedFromDisk const flagsSuspect = this[_flagsSuspect] - const mutateTree = this[_mutateTree] + const mutateTree = this.#mutateTree // if the options set prune:false, then we don't prune, but we still // mark the extraneous items in the tree if we modified it at all. // If we did no modifications, we just iterate over the extraneous nodes. @@ -1481,21 +1476,28 @@ This is a one-time fix-up, please be patient... // then the tree is suspect. Prune what is marked as extraneous. // otherwise, don't bother. const needPrune = metaFromDisk && (mutateTree || flagsSuspect) - if (this[_prune] && needPrune) { + if (this.#prune && needPrune) { this[_idealTreePrune]() + for (const node of this.idealTree.inventory.values()) { + if (node.extraneous) { + node.parent = null + } + } } process.emit('timeEnd', 'idealTree:fixDepFlags') } [_idealTreePrune] () { - for (const node of this.idealTree.inventory.filter(n => n.extraneous)) { - node.parent = null + for (const node of this.idealTree.inventory.values()) { + if (node.extraneous) { + node.parent = null + } } } - [_pruneFailedOptional] () { - for (const node of this[_loadFailures]) { + #pruneFailedOptional () { + for (const node of this.#loadFailures) { if (!node.optional) { throw node.errors[0] } diff --git a/workspaces/arborist/lib/arborist/index.js b/workspaces/arborist/lib/arborist/index.js index ba68e883a4317..ec25117c2a874 100644 --- a/workspaces/arborist/lib/arborist/index.js +++ b/workspaces/arborist/lib/arborist/index.js @@ -37,7 +37,7 @@ const mixins = [ require('./deduper.js'), require('./audit.js'), require('./build-ideal-tree.js'), - require('./load-workspaces.js'), + require('./set-workspaces.js'), require('./load-actual.js'), require('./load-virtual.js'), require('./rebuild.js'), diff --git a/workspaces/arborist/lib/arborist/load-actual.js b/workspaces/arborist/lib/arborist/load-actual.js index a87e09bc05328..8c4e148464d33 100644 --- a/workspaces/arborist/lib/arborist/load-actual.js +++ b/workspaces/arborist/lib/arborist/load-actual.js @@ -17,28 +17,31 @@ const realpath = require('../realpath.js') // public symbols const _changePath = Symbol.for('_changePath') const _global = Symbol.for('global') -const _loadWorkspaces = Symbol.for('loadWorkspaces') +const _setWorkspaces = Symbol.for('setWorkspaces') const _rpcache = Symbol.for('realpathCache') const _stcache = Symbol.for('statCache') -// private symbols -const _actualTree = Symbol('actualTree') -const _actualTreeLoaded = Symbol('actualTreeLoaded') -const _actualTreePromise = Symbol('actualTreePromise') -const _cache = Symbol('nodeLoadingCache') -const _filter = Symbol('filter') -const _findMissingEdges = Symbol('findMissingEdges') -const _loadActual = Symbol('loadActual') -const _loadFSChildren = Symbol('loadFSChildren') -const _loadFSNode = Symbol('loadFSNode') -const _loadFSTree = Symbol('loadFSTree') -const _newLink = Symbol('newLink') -const _newNode = Symbol('newNode') -const _topNodes = Symbol('linkTargets') -const _transplant = Symbol('transplant') -const _transplantFilter = Symbol('transplantFilter') - module.exports = cls => class ActualLoader extends cls { + #actualTree + // ensure when walking the tree that we don't call loadTree on the same + // actual node more than one time. + #actualTreeLoaded = new Set() + #actualTreePromise + + // cache of nodes when loading the actualTree, so that we avoid loaded the + // same node multiple times when symlinks attack. + #cache = new Map() + #filter + + // cache of link targets for setting fsParent links + // We don't do fsParent as a magic getter/setter, because it'd be too costly + // to keep up to date along the walk. + // And, we know that it can ONLY be relevant when the node is a target of a + // link, otherwise it'd be in a node_modules folder, so take advantage of + // that to limit the scans later. + #topNodes = new Set() + #transplantFilter + constructor (options) { super(options) @@ -47,27 +50,11 @@ module.exports = cls => class ActualLoader extends cls { // the tree of nodes on disk this.actualTree = options.actualTree - // ensure when walking the tree that we don't call loadTree on the - // same actual node more than one time. - this[_actualTreeLoaded] = new Set() - // caches for cached realpath calls const cwd = process.cwd() // assume that the cwd is real enough for our purposes this[_rpcache] = new Map([[cwd, cwd]]) this[_stcache] = new Map() - - // cache of nodes when loading the actualTree, so that we avoid - // loaded the same node multiple times when symlinks attack. - this[_cache] = new Map() - - // cache of link targets for setting fsParent links - // We don't do fsParent as a magic getter/setter, because - // it'd be too costly to keep up to date along the walk. - // And, we know that it can ONLY be relevant when the node - // is a target of a link, otherwise it'd be in a node_modules - // folder, so take advantage of that to limit the scans later. - this[_topNodes] = new Set() } // public method @@ -81,12 +68,12 @@ module.exports = cls => class ActualLoader extends cls { if (this.actualTree) { return this.actualTree } - if (!this[_actualTreePromise]) { + if (!this.#actualTreePromise) { // allow the user to set options on the ctor as well. // XXX: deprecate separate method options objects. options = { ...this.options, ...options } - this[_actualTreePromise] = this[_loadActual](options) + this.#actualTreePromise = this.#loadActual(options) .then(tree => { // reset all deps to extraneous prior to recalc if (!options.root) { @@ -102,14 +89,15 @@ module.exports = cls => class ActualLoader extends cls { return this.actualTree }) } - return this[_actualTreePromise] + return this.#actualTreePromise } + // return the promise so that we don't ever have more than one going at the // same time. This is so that buildIdealTree can default to the actualTree // if no shrinkwrap present, but reify() can still call buildIdealTree and // loadActual in parallel safely. - async [_loadActual] (options) { + async #loadActual (options) { // mostly realpath to throw if the root doesn't exist const { global = false, @@ -119,8 +107,8 @@ module.exports = cls => class ActualLoader extends cls { ignoreMissing = false, forceActual = false, } = options - this[_filter] = filter - this[_transplantFilter] = transplantFilter + this.#filter = filter + this.#transplantFilter = transplantFilter if (global) { const real = await realpath(this.path, this[_rpcache], this[_stcache]) @@ -132,19 +120,19 @@ module.exports = cls => class ActualLoader extends cls { loadOverrides: true, } if (this.path === real) { - this[_actualTree] = this[_newNode](params) + this.#actualTree = this.#newNode(params) } else { - this[_actualTree] = await this[_newLink](params) + this.#actualTree = await this.#newLink(params) } } else { // not in global mode, hidden lockfile is allowed, load root pkg too - this[_actualTree] = await this[_loadFSNode]({ + this.#actualTree = await this.#loadFSNode({ path: this.path, real: await realpath(this.path, this[_rpcache], this[_stcache]), loadOverrides: true, }) - this[_actualTree].assertRootOverrides() + this.#actualTree.assertRootOverrides() // if forceActual is set, don't even try the hidden lockfile if (!forceActual) { @@ -152,48 +140,48 @@ module.exports = cls => class ActualLoader extends cls { // in the folder, or if any of the entries in the hidden lockfile are // missing. const meta = await Shrinkwrap.load({ - path: this[_actualTree].path, + path: this.#actualTree.path, hiddenLockfile: true, resolveOptions: this.options, }) if (meta.loadedFromDisk) { - this[_actualTree].meta = meta + this.#actualTree.meta = meta // have to load on a new Arborist object, so we don't assign // the virtualTree on this one! Also, the weird reference is because // we can't easily get a ref to Arborist in this module, without // creating a circular reference, since this class is a mixin used // to build up the Arborist class itself. await new this.constructor({ ...this.options }).loadVirtual({ - root: this[_actualTree], + root: this.#actualTree, }) - await this[_loadWorkspaces](this[_actualTree]) + await this[_setWorkspaces](this.#actualTree) - this[_transplant](root) - return this[_actualTree] + this.#transplant(root) + return this.#actualTree } } const meta = await Shrinkwrap.load({ - path: this[_actualTree].path, + path: this.#actualTree.path, lockfileVersion: this.options.lockfileVersion, resolveOptions: this.options, }) - this[_actualTree].meta = meta + this.#actualTree.meta = meta } - await this[_loadFSTree](this[_actualTree]) - await this[_loadWorkspaces](this[_actualTree]) + await this.#loadFSTree(this.#actualTree) + await this[_setWorkspaces](this.#actualTree) // if there are workspace targets without Link nodes created, load // the targets, so that we know what they are. - if (this[_actualTree].workspaces && this[_actualTree].workspaces.size) { + if (this.#actualTree.workspaces && this.#actualTree.workspaces.size) { const promises = [] - for (const path of this[_actualTree].workspaces.values()) { - if (!this[_cache].has(path)) { + for (const path of this.#actualTree.workspaces.values()) { + if (!this.#cache.has(path)) { // workspace overrides use the root overrides - const p = this[_loadFSNode]({ path, root: this[_actualTree], useRootOverrides: true }) - .then(node => this[_loadFSTree](node)) + const p = this.#loadFSNode({ path, root: this.#actualTree, useRootOverrides: true }) + .then(node => this.#loadFSTree(node)) promises.push(p) } } @@ -201,32 +189,32 @@ module.exports = cls => class ActualLoader extends cls { } if (!ignoreMissing) { - await this[_findMissingEdges]() + await this.#findMissingEdges() } // try to find a node that is the parent in a fs tree sense, but not a // node_modules tree sense, of any link targets. this allows us to // resolve deps that node will find, but a legacy npm view of the // world would not have noticed. - for (const path of this[_topNodes]) { - const node = this[_cache].get(path) + for (const path of this.#topNodes) { + const node = this.#cache.get(path) if (node && !node.parent && !node.fsParent) { for (const p of walkUp(dirname(path))) { - if (this[_cache].has(p)) { - node.fsParent = this[_cache].get(p) + if (this.#cache.has(p)) { + node.fsParent = this.#cache.get(p) break } } } } - this[_transplant](root) + this.#transplant(root) if (global) { // need to depend on the children, or else all of them // will end up being flagged as extraneous, since the // global root isn't a "real" project - const tree = this[_actualTree] + const tree = this.#actualTree const actualRoot = tree.isLink ? tree.target : tree const { dependencies = {} } = actualRoot.package for (const [name, kid] of actualRoot.children.entries()) { @@ -235,30 +223,30 @@ module.exports = cls => class ActualLoader extends cls { } actualRoot.package = { ...actualRoot.package, dependencies } } - return this[_actualTree] + return this.#actualTree } - [_transplant] (root) { - if (!root || root === this[_actualTree]) { + #transplant (root) { + if (!root || root === this.#actualTree) { return } - this[_actualTree][_changePath](root.path) - for (const node of this[_actualTree].children.values()) { - if (!this[_transplantFilter](node)) { + this.#actualTree[_changePath](root.path) + for (const node of this.#actualTree.children.values()) { + if (!this.#transplantFilter(node)) { node.root = null } } - root.replace(this[_actualTree]) - for (const node of this[_actualTree].fsChildren) { - node.root = this[_transplantFilter](node) ? root : null + root.replace(this.#actualTree) + for (const node of this.#actualTree.fsChildren) { + node.root = this.#transplantFilter(node) ? root : null } - this[_actualTree] = root + this.#actualTree = root } - async [_loadFSNode] ({ path, parent, real, root, loadOverrides, useRootOverrides }) { + async #loadFSNode ({ path, parent, real, root, loadOverrides, useRootOverrides }) { if (!real) { try { real = await realpath(path, this[_rpcache], this[_stcache]) @@ -275,7 +263,7 @@ module.exports = cls => class ActualLoader extends cls { } } - const cached = this[_cache].get(path) + const cached = this.#cache.get(path) let node // missing edges get a dummy node, assign the parent and return it if (cached && !cached.dummy) { @@ -306,67 +294,67 @@ module.exports = cls => class ActualLoader extends cls { // Node which will attach it to its errors array (Link passes it along to // its target node) if (normalize(path) === real) { - node = this[_newNode](params) + node = this.#newNode(params) } else { - node = await this[_newLink](params) + node = await this.#newLink(params) } } - this[_cache].set(path, node) + this.#cache.set(path, node) return node } - [_newNode] (options) { + #newNode (options) { // check it for an fsParent if it's a tree top. there's a decent chance // it'll get parented later, making the fsParent scan a no-op, but better // safe than sorry, since it's cheap. const { parent, realpath } = options if (!parent) { - this[_topNodes].add(realpath) + this.#topNodes.add(realpath) } return new Node(options) } - async [_newLink] (options) { + async #newLink (options) { const { realpath } = options - this[_topNodes].add(realpath) - const target = this[_cache].get(realpath) + this.#topNodes.add(realpath) + const target = this.#cache.get(realpath) const link = new Link({ ...options, target }) if (!target) { // Link set its target itself in this case - this[_cache].set(realpath, link.target) + this.#cache.set(realpath, link.target) // if a link target points at a node outside of the root tree's // node_modules hierarchy, then load that node as well. - await this[_loadFSTree](link.target) + await this.#loadFSTree(link.target) } return link } - async [_loadFSTree] (node) { - const did = this[_actualTreeLoaded] + async #loadFSTree (node) { + const did = this.#actualTreeLoaded if (!did.has(node.target.realpath)) { did.add(node.target.realpath) - await this[_loadFSChildren](node.target) + await this.#loadFSChildren(node.target) return Promise.all( [...node.target.children.entries()] .filter(([name, kid]) => !did.has(kid.realpath)) - .map(([name, kid]) => this[_loadFSTree](kid)) + .map(([name, kid]) => this.#loadFSTree(kid)) ) } } // create child nodes for all the entries in node_modules // and attach them to the node as a parent - async [_loadFSChildren] (node) { + async #loadFSChildren (node) { const nm = resolve(node.realpath, 'node_modules') try { const kids = await readdirScoped(nm).then(paths => paths.map(p => p.replace(/\\/g, '/'))) return Promise.all( // ignore . dirs and retired scoped package folders kids.filter(kid => !/^(@[^/]+\/)?\./.test(kid)) - .filter(kid => this[_filter](node, kid)) - .map(kid => this[_loadFSNode]({ + .filter(kid => this.#filter(node, kid)) + .map(kid => this.#loadFSNode({ parent: node, path: resolve(nm, kid), }))) @@ -375,7 +363,7 @@ module.exports = cls => class ActualLoader extends cls { } } - async [_findMissingEdges] () { + async #findMissingEdges () { // try to resolve any missing edges by walking up the directory tree, // checking for the package in each node_modules folder. stop at the // root directory. @@ -385,7 +373,7 @@ module.exports = cls => class ActualLoader extends cls { // because people sometimes develop in ~/projects/node_modules/... // so we'd end up loading a massive tree with lots of unrelated junk. const nmContents = new Map() - const tree = this[_actualTree] + const tree = this.#actualTree for (const node of tree.inventory.values()) { const ancestor = ancestorPath(node.realpath, this.path) @@ -410,28 +398,37 @@ module.exports = cls => class ActualLoader extends cls { break } - const entries = nmContents.get(p) || await readdirScoped(p + '/node_modules') - .catch(() => []).then(paths => paths.map(p => p.replace(/\\/g, '/'))) - nmContents.set(p, entries) + let entries + if (!nmContents.has(p)) { + entries = await readdirScoped(p + '/node_modules') + .catch(() => []).then(paths => paths.map(p => p.replace(/\\/g, '/'))) + nmContents.set(p, entries) + } else { + entries = nmContents.get(p) + } + if (!entries.includes(name)) { continue } - const d = this[_cache].has(p) ? await this[_cache].get(p) - : new Node({ path: p, root: node.root, dummy: true }) - // not a promise - this[_cache].set(p, d) + let d + if (!this.#cache.has(p)) { + d = new Node({ path: p, root: node.root, dummy: true }) + this.#cache.set(p, d) + } else { + d = this.#cache.get(p) + } if (d.dummy) { // it's a placeholder, so likely would not have loaded this dep, // unless another dep in the tree also needs it. const depPath = normalize(`${p}/node_modules/${name}`) - const cached = this[_cache].get(depPath) + const cached = this.#cache.get(depPath) if (!cached || cached.dummy) { - depPromises.push(this[_loadFSNode]({ + depPromises.push(this.#loadFSNode({ path: depPath, root: node.root, parent: d, - }).then(node => this[_loadFSTree](node))) + }).then(node => this.#loadFSTree(node))) } } break diff --git a/workspaces/arborist/lib/arborist/load-virtual.js b/workspaces/arborist/lib/arborist/load-virtual.js index b2a6ec2315a4f..9b681a47a8358 100644 --- a/workspaces/arborist/lib/arborist/load-virtual.js +++ b/workspaces/arborist/lib/arborist/load-virtual.js @@ -1,5 +1,4 @@ // mixin providing the loadVirtual method -const localeCompare = require('@isaacs/string-locale-compare')('en') const mapWorkspaces = require('@npmcli/map-workspaces') const { resolve } = require('path') @@ -14,23 +13,12 @@ const calcDepFlags = require('../calc-dep-flags.js') const rpj = require('read-package-json-fast') const treeCheck = require('../tree-check.js') -const loadFromShrinkwrap = Symbol('loadFromShrinkwrap') -const resolveNodes = Symbol('resolveNodes') -const resolveLinks = Symbol('resolveLinks') -const assignBundles = Symbol('assignBundles') -const loadRoot = Symbol('loadRoot') -const loadNode = Symbol('loadVirtualNode') -const loadLink = Symbol('loadVirtualLink') -const loadWorkspaces = Symbol.for('loadWorkspaces') const flagsSuspect = Symbol.for('flagsSuspect') -const reCalcDepFlags = Symbol('reCalcDepFlags') -const checkRootEdges = Symbol('checkRootEdges') -const rootOptionProvided = Symbol('rootOptionProvided') - -const depsToEdges = (type, deps) => - Object.entries(deps).map(d => [type, ...d]) +const setWorkspaces = Symbol.for('setWorkspaces') module.exports = cls => class VirtualLoader extends cls { + #rootOptionProvided + constructor (options) { super(options) @@ -50,7 +38,7 @@ module.exports = cls => class VirtualLoader extends cls { options = { ...this.options, ...options } if (options.root && options.root.meta) { - await this[loadFromShrinkwrap](options.root.meta, options.root) + await this.#loadFromShrinkwrap(options.root.meta, options.root) return treeCheck(this.virtualTree) } @@ -67,24 +55,24 @@ module.exports = cls => class VirtualLoader extends cls { // when building the ideal tree, we pass in a root node to this function // otherwise, load it from the root package json or the lockfile const { - root = await this[loadRoot](s), + root = await this.#loadRoot(s), } = options - this[rootOptionProvided] = options.root + this.#rootOptionProvided = options.root - await this[loadFromShrinkwrap](s, root) + await this.#loadFromShrinkwrap(s, root) root.assertRootOverrides() return treeCheck(this.virtualTree) } - async [loadRoot] (s) { + async #loadRoot (s) { const pj = this.path + '/package.json' const pkg = await rpj(pj).catch(() => s.data.packages['']) || {} - return this[loadWorkspaces](this[loadNode]('', pkg, true)) + return this[setWorkspaces](this.#loadNode('', pkg, true)) } - async [loadFromShrinkwrap] (s, root) { - if (!this[rootOptionProvided]) { + async #loadFromShrinkwrap (s, root) { + if (!this.#rootOptionProvided) { // root is never any of these things, but might be a brand new // baby Node object that never had its dep flags calculated. root.extraneous = false @@ -96,41 +84,37 @@ module.exports = cls => class VirtualLoader extends cls { this[flagsSuspect] = true } - this[checkRootEdges](s, root) + this.#checkRootEdges(s, root) root.meta = s this.virtualTree = root - const { links, nodes } = this[resolveNodes](s, root) - await this[resolveLinks](links, nodes) + const { links, nodes } = this.#resolveNodes(s, root) + await this.#resolveLinks(links, nodes) if (!(s.originalLockfileVersion >= 2)) { - this[assignBundles](nodes) + this.#assignBundles(nodes) } if (this[flagsSuspect]) { - this[reCalcDepFlags](nodes.values()) - } - return root - } - - [reCalcDepFlags] (nodes) { - // reset all dep flags - // can't use inventory here, because virtualTree might not be root - for (const node of nodes) { - if (node.isRoot || node === this[rootOptionProvided]) { - continue + // reset all dep flags + // can't use inventory here, because virtualTree might not be root + for (const node of nodes.values()) { + if (node.isRoot || node === this.#rootOptionProvided) { + continue + } + node.extraneous = true + node.dev = true + node.optional = true + node.devOptional = true + node.peer = true } - node.extraneous = true - node.dev = true - node.optional = true - node.devOptional = true - node.peer = true + calcDepFlags(this.virtualTree, !this.#rootOptionProvided) } - calcDepFlags(this.virtualTree, !this[rootOptionProvided]) + return root } // check the lockfile deps, and see if they match. if they do not // then we have to reset dep flags at the end. for example, if the // user manually edits their package.json file, then we need to know // that the idealTree is no longer entirely trustworthy. - [checkRootEdges] (s, root) { + #checkRootEdges (s, root) { // loaded virtually from tree, no chance of being out of sync // ancient lockfiles are critically damaged by this process, // so we need to just hope for the best in those cases. @@ -144,6 +128,7 @@ module.exports = cls => class VirtualLoader extends cls { const optional = lock.optionalDependencies || {} const peer = lock.peerDependencies || {} const peerOptional = {} + if (lock.peerDependenciesMeta) { for (const [name, meta] of Object.entries(lock.peerDependenciesMeta)) { if (meta.optional && peer[name] !== undefined) { @@ -152,50 +137,45 @@ module.exports = cls => class VirtualLoader extends cls { } } } + for (const name of Object.keys(optional)) { delete prod[name] } - const lockWS = [] + const lockWS = {} const workspaces = mapWorkspaces.virtual({ cwd: this.path, lockfile: s.data, }) + for (const [name, path] of workspaces.entries()) { - lockWS.push(['workspace', name, `file:${path.replace(/#/g, '%23')}`]) + lockWS[name] = `file:${path.replace(/#/g, '%23')}` } - const lockEdges = [ - ...depsToEdges('prod', prod), - ...depsToEdges('dev', dev), - ...depsToEdges('optional', optional), - ...depsToEdges('peer', peer), - ...depsToEdges('peerOptional', peerOptional), - ...lockWS, - ].sort(([atype, aname], [btype, bname]) => - localeCompare(atype, btype) || localeCompare(aname, bname)) - - const rootEdges = [...root.edgesOut.values()] - .map(e => [e.type, e.name, e.spec]) - .sort(([atype, aname], [btype, bname]) => - localeCompare(atype, btype) || localeCompare(aname, bname)) - - if (rootEdges.length !== lockEdges.length) { - // something added or removed - return this[flagsSuspect] = true - } + // Should rootNames exclude optional? + const rootNames = new Set(root.edgesOut.keys()) - for (let i = 0; i < lockEdges.length; i++) { - if (rootEdges[i][0] !== lockEdges[i][0] || - rootEdges[i][1] !== lockEdges[i][1] || - rootEdges[i][2] !== lockEdges[i][2]) { - return this[flagsSuspect] = true + const lockByType = ({ dev, optional, peer, peerOptional, prod, workspace: lockWS }) + + // Find anything in shrinkwrap deps that doesn't match root's type or spec + for (const type in lockByType) { + const deps = lockByType[type] + for (const name in deps) { + const edge = root.edgesOut.get(name) + if (!edge || edge.type !== type || edge.spec !== deps[name]) { + return this[flagsSuspect] = true + } + rootNames.delete(name) } } + // Something was in root that's not accounted for in shrinkwrap + if (rootNames.size) { + return this[flagsSuspect] = true + } } // separate out link metadatas, and create Node objects for nodes - [resolveNodes] (s, root) { + #resolveNodes (s, root) { const links = new Map() const nodes = new Map([['', root]]) for (const [location, meta] of Object.entries(s.data.packages)) { @@ -207,7 +187,7 @@ module.exports = cls => class VirtualLoader extends cls { if (meta.link) { links.set(location, meta) } else { - nodes.set(location, this[loadNode](location, meta)) + nodes.set(location, this.#loadNode(location, meta)) } } return { links, nodes } @@ -215,12 +195,12 @@ module.exports = cls => class VirtualLoader extends cls { // links is the set of metadata, and nodes is the map of non-Link nodes // Set the targets to nodes in the set, if we have them (we might not) - async [resolveLinks] (links, nodes) { + async #resolveLinks (links, nodes) { for (const [location, meta] of links.entries()) { const targetPath = resolve(this.path, meta.resolved) const targetLoc = relpath(this.path, targetPath) const target = nodes.get(targetLoc) - const link = this[loadLink](location, targetLoc, target, meta) + const link = this.#loadLink(location, targetLoc, target, meta) nodes.set(location, link) nodes.set(targetLoc, link.target) @@ -236,7 +216,7 @@ module.exports = cls => class VirtualLoader extends cls { } } - [assignBundles] (nodes) { + #assignBundles (nodes) { for (const [location, node] of nodes) { // Skip assignment of parentage for the root package if (!location || node.isLink && !node.target.location) { @@ -265,7 +245,7 @@ module.exports = cls => class VirtualLoader extends cls { } } - [loadNode] (location, sw, loadOverrides) { + #loadNode (location, sw, loadOverrides) { const p = this.virtualTree ? this.virtualTree.realpath : this.path const path = resolve(p, location) // shrinkwrap doesn't include package name unless necessary @@ -303,7 +283,7 @@ module.exports = cls => class VirtualLoader extends cls { return node } - [loadLink] (location, targetLoc, target, meta) { + #loadLink (location, targetLoc, target, meta) { const path = resolve(this.path, location) const link = new Link({ installLinks: this.installLinks, diff --git a/workspaces/arborist/lib/arborist/load-workspaces.js b/workspaces/arborist/lib/arborist/set-workspaces.js similarity index 80% rename from workspaces/arborist/lib/arborist/load-workspaces.js rename to workspaces/arborist/lib/arborist/set-workspaces.js index effa5a0cda24f..27a12708a7e82 100644 --- a/workspaces/arborist/lib/arborist/load-workspaces.js +++ b/workspaces/arborist/lib/arborist/set-workspaces.js @@ -1,10 +1,10 @@ const mapWorkspaces = require('@npmcli/map-workspaces') // shared ref used by other mixins/Arborist -const _loadWorkspaces = Symbol.for('loadWorkspaces') +const _setWorkspaces = Symbol.for('setWorkspaces') module.exports = cls => class MapWorkspaces extends cls { - async [_loadWorkspaces] (node) { + async [_setWorkspaces] (node) { const workspaces = await mapWorkspaces({ cwd: node.path, pkg: node.package, diff --git a/workspaces/arborist/lib/calc-dep-flags.js b/workspaces/arborist/lib/calc-dep-flags.js index 2f33981419d0c..45ed9562479af 100644 --- a/workspaces/arborist/lib/calc-dep-flags.js +++ b/workspaces/arborist/lib/calc-dep-flags.js @@ -51,14 +51,12 @@ const calcDepFlagsStep = (node) => { // however, for convenience and to save an extra rewalk, we leave // it set when we are in *either* tree, and then omit it from the // package-lock if either dev or optional are set. - const unsetDevOpt = !node.devOptional && !node.dev && !node.optional && - !dev && !optional + const unsetDevOpt = !node.devOptional && !node.dev && !node.optional && !dev && !optional // if we are not in the devOpt tree, then we're also not in // either the dev or opt trees const unsetDev = unsetDevOpt || !node.dev && !dev - const unsetOpt = unsetDevOpt || - !node.optional && !optional + const unsetOpt = unsetDevOpt || !node.optional && !optional const unsetPeer = !node.peer && !peer if (unsetPeer) { @@ -91,8 +89,7 @@ const resetParents = (node, flag) => { } } -// typically a short walk, since it only traverses deps that -// have the flag set. +// typically a short walk, since it only traverses deps that have the flag set. const unsetFlag = (node, flag) => { if (node[flag]) { node[flag] = false @@ -104,10 +101,17 @@ const unsetFlag = (node, flag) => { node.target.extraneous = node.target[flag] = false } }, - getChildren: node => [...node.target.edgesOut.values()] - .filter(edge => edge.to && edge.to[flag] && - (flag !== 'peer' && edge.type === 'peer' || edge.type === 'prod')) - .map(edge => edge.to), + getChildren: node => { + const children = [] + for (const edge of node.target.edgesOut.values()) { + if (edge.to && edge.to[flag] && + (flag !== 'peer' && edge.type === 'peer' || edge.type === 'prod') + ) { + children.push(edge.to) + } + } + return children + }, }) } } diff --git a/workspaces/arborist/lib/dep-valid.js b/workspaces/arborist/lib/dep-valid.js index 9c1bc7d3f23e9..4afb5e47cf111 100644 --- a/workspaces/arborist/lib/dep-valid.js +++ b/workspaces/arborist/lib/dep-valid.js @@ -84,15 +84,21 @@ const depValid = (child, requested, requestor) => { const reqHost = requested.hosted const reqCommit = /^[a-fA-F0-9]{40}$/.test(requested.gitCommittish || '') const nc = { noCommittish: !reqCommit } - const sameRepo = - resHost ? reqHost && reqHost.ssh(nc) === resHost.ssh(nc) - : resRepo.fetchSpec === requested.fetchSpec - - return !sameRepo ? false - : !requested.gitRange ? true - : semver.satisfies(child.package.version, requested.gitRange, { - loose: true, - }) + if (!resHost) { + if (resRepo.fetchSpec !== requested.fetchSpec) { + return false + } + } else { + if (reqHost?.ssh(nc) !== resHost.ssh(nc)) { + return false + } + } + if (!requested.gitRange) { + return true + } + return semver.satisfies(child.package.version, requested.gitRange, { + loose: true, + }) } default: // unpossible, just being cautious diff --git a/workspaces/arborist/lib/edge.js b/workspaces/arborist/lib/edge.js index 84078ba050d10..cc9698ad6cae7 100644 --- a/workspaces/arborist/lib/edge.js +++ b/workspaces/arborist/lib/edge.js @@ -4,194 +4,210 @@ const util = require('util') const npa = require('npm-package-arg') const depValid = require('./dep-valid.js') -const _from = Symbol('_from') -const _to = Symbol('_to') -const _type = Symbol('_type') -const _spec = Symbol('_spec') -const _accept = Symbol('_accept') -const _name = Symbol('_name') -const _error = Symbol('_error') -const _loadError = Symbol('_loadError') -const _setFrom = Symbol('_setFrom') -const _explain = Symbol('_explain') -const _explanation = Symbol('_explanation') - -const types = new Set([ - 'prod', - 'dev', - 'optional', - 'peer', - 'peerOptional', - 'workspace', -]) - -class ArboristEdge {} -const printableEdge = (edge) => { - const edgeFrom = edge.from && edge.from.location - const edgeTo = edge.to && edge.to.location - const override = edge.overrides && edge.overrides.value - - return Object.assign(new ArboristEdge(), { - name: edge.name, - spec: edge.spec, - type: edge.type, - ...(edgeFrom != null ? { from: edgeFrom } : {}), - ...(edgeTo ? { to: edgeTo } : {}), - ...(edge.error ? { error: edge.error } : {}), - ...(edge.peerConflicted ? { peerConflicted: true } : {}), - ...(override ? { overridden: override } : {}), - }) + +class ArboristEdge { + constructor (edge) { + this.name = edge.name + this.spec = edge.spec + this.type = edge.type + + const edgeFrom = edge.from?.location + const edgeTo = edge.to?.location + const override = edge.overrides?.value + + if (edgeFrom != null) { + this.from = edgeFrom + } + if (edgeTo) { + this.to = edgeTo + } + if (edge.error) { + this.error = edge.error + } + if (edge.peerConflicted) { + this.peerConflicted = true + } + if (override) { + this.overridden = override + } + } } class Edge { + #accept + #error + #explanation + #from + #name + #spec + #to + #type + + static types = Object.freeze([ + 'prod', + 'dev', + 'optional', + 'peer', + 'peerOptional', + 'workspace', + ]) + + // XXX where is this used? + static errors = Object.freeze([ + 'DETACHED', + 'MISSING', + 'PEER LOCAL', + 'INVALID', + ]) + constructor (options) { const { type, name, spec, accept, from, overrides } = options + // XXX are all of these error states even possible? if (typeof spec !== 'string') { throw new TypeError('must provide string spec') } - + if (!Edge.types.includes(type)) { + throw new TypeError(`invalid type: ${type}\n(valid types are: ${Edge.types.join(', ')})`) + } if (type === 'workspace' && npa(spec).type !== 'directory') { throw new TypeError('workspace edges must be a symlink') } - - this[_spec] = spec - - if (overrides !== undefined) { - this.overrides = overrides + if (typeof name !== 'string') { + throw new TypeError('must provide dependency name') + } + if (!from) { + throw new TypeError('must provide "from" node') } - if (accept !== undefined) { if (typeof accept !== 'string') { throw new TypeError('accept field must be a string if provided') } - this[_accept] = accept || '*' + this.#accept = accept || '*' } - - if (typeof name !== 'string') { - throw new TypeError('must provide dependency name') + if (overrides !== undefined) { + this.overrides = overrides } - this[_name] = name - if (!types.has(type)) { - throw new TypeError( - `invalid type: ${type}\n` + - `(valid types are: ${Edge.types.join(', ')})`) - } - this[_type] = type - if (!from) { - throw new TypeError('must provide "from" node') - } - this[_setFrom](from) - this[_error] = this[_loadError]() + this.#name = name + this.#type = type + this.#spec = spec + this.#explanation = null + this.#from = from + + from.edgesOut.get(this.#name)?.detach() + from.addEdgeOut(this) + + this.reload(true) this.peerConflicted = false } satisfiedBy (node) { - if (node.name !== this.name) { + if (node.name !== this.#name) { return false } // NOTE: this condition means we explicitly do not support overriding // bundled or shrinkwrapped dependencies - const spec = (node.hasShrinkwrap || node.inShrinkwrap || node.inBundle) - ? this.rawSpec - : this.spec - return depValid(node, spec, this.accept, this.from) - } - - explain (seen = []) { - if (this[_explanation]) { - return this[_explanation] + if (node.hasShrinkwrap || node.inShrinkwrap || node.inBundle) { + return depValid(node, this.rawSpec, this.#accept, this.#from) } - - return this[_explanation] = this[_explain](seen) + return depValid(node, this.spec, this.#accept, this.#from) } // return the edge data, and an explanation of how that edge came to be here - [_explain] (seen) { - const { error, from, bundled } = this - return { - type: this.type, - name: this.name, - spec: this.spec, - ...(this.rawSpec !== this.spec ? { - rawSpec: this.rawSpec, - overridden: true, - } : {}), - ...(bundled ? { bundled } : {}), - ...(error ? { error } : {}), - ...(from ? { from: from.explain(null, seen) } : {}), + explain (seen = []) { + if (!this.#explanation) { + const explanation = { + type: this.#type, + name: this.#name, + spec: this.spec, + } + if (this.rawSpec !== this.spec) { + explanation.rawSpec = this.rawSpec + explanation.overridden = true + } + if (this.bundled) { + explanation.bundled = this.bundled + } + if (this.error) { + explanation.error = this.error + } + if (this.#from) { + explanation.from = this.#from.explain(null, seen) + } + this.#explanation = explanation } + return this.#explanation } get bundled () { - if (!this.from) { - return false - } - const { package: { bundleDependencies = [] } } = this.from - return bundleDependencies.includes(this.name) + return !!this.#from?.package?.bundleDependencies?.includes(this.#name) } get workspace () { - return this[_type] === 'workspace' + return this.#type === 'workspace' } get prod () { - return this[_type] === 'prod' + return this.#type === 'prod' } get dev () { - return this[_type] === 'dev' + return this.#type === 'dev' } get optional () { - return this[_type] === 'optional' || this[_type] === 'peerOptional' + return this.#type === 'optional' || this.#type === 'peerOptional' } get peer () { - return this[_type] === 'peer' || this[_type] === 'peerOptional' + return this.#type === 'peer' || this.#type === 'peerOptional' } get type () { - return this[_type] + return this.#type } get name () { - return this[_name] + return this.#name } get rawSpec () { - return this[_spec] + return this.#spec } get spec () { - if (this.overrides?.value && this.overrides.value !== '*' && this.overrides.name === this.name) { + if (this.overrides?.value && this.overrides.value !== '*' && this.overrides.name === this.#name) { if (this.overrides.value.startsWith('$')) { const ref = this.overrides.value.slice(1) // we may be a virtual root, if we are we want to resolve reference overrides // from the real root, not the virtual one - const pkg = this.from.sourceReference - ? this.from.sourceReference.root.package - : this.from.root.package - const overrideSpec = (pkg.devDependencies && pkg.devDependencies[ref]) || - (pkg.optionalDependencies && pkg.optionalDependencies[ref]) || - (pkg.dependencies && pkg.dependencies[ref]) || - (pkg.peerDependencies && pkg.peerDependencies[ref]) - - if (overrideSpec) { - return overrideSpec + const pkg = this.#from.sourceReference + ? this.#from.sourceReference.root.package + : this.#from.root.package + if (pkg.devDependencies?.[ref]) { + return pkg.devDependencies[ref] + } + if (pkg.optionalDependencies?.[ref]) { + return pkg.optionalDependencies[ref] + } + if (pkg.dependencies?.[ref]) { + return pkg.dependencies[ref] + } + if (pkg.peerDependencies?.[ref]) { + return pkg.peerDependencies[ref] } throw new Error(`Unable to resolve reference ${this.overrides.value}`) } return this.overrides.value } - return this[_spec] + return this.#spec } get accept () { - return this[_accept] + return this.#accept } get valid () { @@ -211,71 +227,70 @@ class Edge { } get error () { - this[_error] = this[_error] || this[_loadError]() - return this[_error] === 'OK' ? null : this[_error] - } - - [_loadError] () { - return !this[_to] ? (this.optional ? null : 'MISSING') - : this.peer && this.from === this.to.parent && !this.from.isTop ? 'PEER LOCAL' - : !this.satisfiedBy(this.to) ? 'INVALID' - : 'OK' + if (!this.#error) { + if (!this.#to) { + if (this.optional) { + this.#error = null + } else { + this.#error = 'MISSING' + } + } else if (this.peer && this.#from === this.#to.parent && !this.#from.isTop) { + this.#error = 'PEER LOCAL' + } else if (!this.satisfiedBy(this.#to)) { + this.#error = 'INVALID' + } else { + this.#error = 'OK' + } + } + if (this.#error === 'OK') { + return null + } + return this.#error } reload (hard = false) { - this[_explanation] = null - if (this[_from].overrides) { - this.overrides = this[_from].overrides.getEdgeRule(this) + this.#explanation = null + if (this.#from.overrides) { + this.overrides = this.#from.overrides.getEdgeRule(this) } else { delete this.overrides } - const newTo = this[_from].resolve(this.name) - if (newTo !== this[_to]) { - if (this[_to]) { - this[_to].edgesIn.delete(this) + const newTo = this.#from.resolve(this.#name) + if (newTo !== this.#to) { + if (this.#to) { + this.#to.edgesIn.delete(this) } - this[_to] = newTo - this[_error] = this[_loadError]() - if (this[_to]) { - this[_to].addEdgeIn(this) + this.#to = newTo + this.#error = null + if (this.#to) { + this.#to.addEdgeIn(this) } } else if (hard) { - this[_error] = this[_loadError]() + this.#error = null } } detach () { - this[_explanation] = null - if (this[_to]) { - this[_to].edgesIn.delete(this) + this.#explanation = null + if (this.#to) { + this.#to.edgesIn.delete(this) } - this[_from].edgesOut.delete(this.name) - this[_to] = null - this[_error] = 'DETACHED' - this[_from] = null - } - - [_setFrom] (node) { - this[_explanation] = null - this[_from] = node - if (node.edgesOut.has(this.name)) { - node.edgesOut.get(this.name).detach() - } - - node.addEdgeOut(this) - this.reload() + this.#from.edgesOut.delete(this.#name) + this.#to = null + this.#error = 'DETACHED' + this.#from = null } get from () { - return this[_from] + return this.#from } get to () { - return this[_to] + return this.#to } toJSON () { - return printableEdge(this) + return new ArboristEdge(this) } [util.inspect.custom] () { @@ -283,12 +298,4 @@ class Edge { } } -Edge.types = [...types] -Edge.errors = [ - 'DETACHED', - 'MISSING', - 'PEER LOCAL', - 'INVALID', -] - module.exports = Edge diff --git a/workspaces/arborist/lib/from-path.js b/workspaces/arborist/lib/from-path.js index 1006f73af3d07..761f15125b343 100644 --- a/workspaces/arborist/lib/from-path.js +++ b/workspaces/arborist/lib/from-path.js @@ -1,24 +1,30 @@ -// file dependencies need their dependencies resolved based on the -// location where the tarball was found, not the location where they -// end up getting installed. directory (ie, symlink) deps also need -// to be resolved based on their targets, but that's what realpath is +// file dependencies need their dependencies resolved based on the location +// where the tarball was found, not the location where they end up getting +// installed. directory (ie, symlink) deps also need to be resolved based on +// their targets, but that's what realpath is const { dirname } = require('path') const npa = require('npm-package-arg') -const fromPath = (node, spec, edge) => { +const fromPath = (node, edge) => { if (edge && edge.overrides && edge.overrides.name === edge.name && edge.overrides.value) { - // fromPath could be called with a node that has a virtual root, if that happens - // we want to make sure we get the real root node when overrides are in use. this - // is to allow things like overriding a dependency with a tarball file that's a - // relative path from the project root - return node.sourceReference - ? node.sourceReference.root.realpath - : node.root.realpath + // fromPath could be called with a node that has a virtual root, if that + // happens we want to make sure we get the real root node when overrides + // are in use. this is to allow things like overriding a dependency with a + // tarball file that's a relative path from the project root + if (node.sourceReference) { + return node.sourceReference.root.realpath + } + return node.root.realpath } - return spec && spec.type === 'file' ? dirname(spec.fetchSpec) - : node.realpath + if (node.resolved) { + const spec = npa(node.resolved) + if (spec?.type === 'file') { + return dirname(spec.fetchSpec) + } + } + return node.realpath } -module.exports = (node, edge) => fromPath(node, node.resolved && npa(node.resolved), edge) +module.exports = fromPath diff --git a/workspaces/arborist/lib/inventory.js b/workspaces/arborist/lib/inventory.js index 34b6f98a8b286..0885034666b50 100644 --- a/workspaces/arborist/lib/inventory.js +++ b/workspaces/arborist/lib/inventory.js @@ -1,45 +1,28 @@ -// a class to manage an inventory and set of indexes of -// a set of objects based on specific fields. -// primary is the primary index key. -// keys is the set of fields to be able to query. -const _primaryKey = Symbol('_primaryKey') -const _index = Symbol('_index') -const defaultKeys = ['name', 'license', 'funding', 'realpath', 'packageName'] +// a class to manage an inventory and set of indexes of a set of objects based +// on specific fields. const { hasOwnProperty } = Object.prototype const debug = require('./debug.js') -// handling for the outdated "licenses" array, just pick the first one -// also support the alternative spelling "licence" -const getLicense = pkg => { - if (pkg) { - const lic = pkg.license || pkg.licence - if (lic) { - return lic - } - const lics = pkg.licenses || pkg.licences - if (Array.isArray(lics)) { - return lics[0] - } - } -} - +const keys = ['name', 'license', 'funding', 'realpath', 'packageName'] class Inventory extends Map { - constructor (opt = {}) { - const { primary, keys } = opt + #index + + constructor () { super() - this[_primaryKey] = primary || 'location' - this[_index] = (keys || defaultKeys).reduce((index, i) => { - index.set(i, new Map()) - return index - }, new Map()) + this.#index = new Map() + for (const key of keys) { + this.#index.set(key, new Map()) + } } + // XXX where is this used? get primaryKey () { - return this[_primaryKey] + return 'location' } + // XXX where is this used? get indexes () { - return [...this[_index].keys()] + return [...keys] } * filter (fn) { @@ -63,28 +46,49 @@ class Inventory extends Map { return } - const current = super.get(node[this.primaryKey]) + const current = super.get(node.location) if (current) { if (current === node) { return } this.delete(current) } - super.set(node[this.primaryKey], node) - for (const [key, map] of this[_index].entries()) { - // if the node has the value, but it's false, then use that - const val_ = hasOwnProperty.call(node, key) ? node[key] - : key === 'license' ? getLicense(node.package) - : node[key] ? node[key] - : node.package && node.package[key] - const val = typeof val_ === 'string' ? val_ - : !val_ || typeof val_ !== 'object' ? val_ - : key === 'license' ? val_.type - : key === 'funding' ? val_.url - : /* istanbul ignore next - not used */ val_ - const set = map.get(val) || new Set() - set.add(node) - map.set(val, set) + super.set(node.location, node) + for (const [key, map] of this.#index.entries()) { + let val + if (hasOwnProperty.call(node, key)) { + // if the node has the value, use it even if it's false + val = node[key] + } else if (key === 'license' && node.package) { + // handling for the outdated "licenses" array, just pick the first one + // also support the alternative spelling "licence" + if (node.package.license) { + val = node.package.license + } else if (node.package.licence) { + val = node.package.licence + } else if (Array.isArray(node.package.licenses)) { + val = node.package.licenses[0] + } else if (Array.isArray(node.package.licences)) { + val = node.package.licences[0] + } + } else if (node[key]) { + val = node[key] + } else { + val = node.package?.[key] + } + if (val && typeof val === 'object') { + // We currently only use license and funding + /* istanbul ignore next - not used */ + if (key === 'license') { + val = val.type + } else if (key === 'funding') { + val = val.url + } + } + if (!map.has(val)) { + map.set(val, new Set()) + } + map.get(val).add(node) } } @@ -93,10 +97,14 @@ class Inventory extends Map { return } - super.delete(node[this.primaryKey]) - for (const [key, map] of this[_index].entries()) { - const val = node[key] !== undefined ? node[key] - : (node[key] || (node.package && node.package[key])) + super.delete(node.location) + for (const [key, map] of this.#index.entries()) { + let val + if (node[key] !== undefined) { + val = node[key] + } else { + val = node.package?.[key] + } const set = map.get(val) if (set) { set.delete(node) @@ -108,13 +116,18 @@ class Inventory extends Map { } query (key, val) { - const map = this[_index].get(key) - return map && (arguments.length === 2 ? map.get(val) : map.keys()) || - new Set() + const map = this.#index.get(key) + if (arguments.length === 2) { + if (map.has(val)) { + return map.get(val) + } + return new Set() + } + return map.keys() } has (node) { - return super.get(node[this.primaryKey]) === node + return super.get(node.location) === node } set (k, v) { diff --git a/workspaces/arborist/lib/node.js b/workspaces/arborist/lib/node.js index e264194fd6a6c..bdc021b7c12a9 100644 --- a/workspaces/arborist/lib/node.js +++ b/workspaces/arborist/lib/node.js @@ -47,21 +47,15 @@ const _package = Symbol('_package') const _parent = Symbol('_parent') const _target = Symbol.for('_target') const _fsParent = Symbol('_fsParent') -const _loadDepType = Symbol('_loadDepType') -const _loadWorkspaces = Symbol('_loadWorkspaces') const _reloadNamedEdges = Symbol('_reloadNamedEdges') // overridden by Link class const _loadDeps = Symbol.for('Arborist.Node._loadDeps') -const _root = Symbol('_root') const _refreshLocation = Symbol.for('_refreshLocation') const _changePath = Symbol.for('_changePath') // used by Link class as well const _delistFromMeta = Symbol.for('_delistFromMeta') -const _global = Symbol.for('global') -const _workspaces = Symbol('_workspaces') const _explain = Symbol('_explain') const _explanation = Symbol('_explanation') -const _meta = Symbol('_meta') const relpath = require('./relpath.js') const consistentResolve = require('./consistent-resolve.js') @@ -72,6 +66,11 @@ const CaseInsensitiveMap = require('./case-insensitive-map.js') const querySelectorAll = require('./query-selector-all.js') class Node { + #global + #meta + #root + #workspaces + constructor (options) { // NB: path can be null if it's a link target const { @@ -109,9 +108,9 @@ class Node { this.queryContext = {} // true if part of a global install - this[_global] = global + this.#global = global - this[_workspaces] = null + this.#workspaces = null this.errors = error ? [error] : [] this.isInStore = isInStore @@ -165,7 +164,7 @@ class Node { this.children = new CaseInsensitiveMap() this.fsChildren = new Set() - this.inventory = new Inventory({}) + this.inventory = new Inventory() this.tops = new Set() this.linksIn = new Set(linksIn || []) @@ -262,18 +261,21 @@ class Node { } get meta () { - return this[_meta] + return this.#meta } set meta (meta) { - this[_meta] = meta + this.#meta = meta if (meta) { meta.add(this) } } get global () { - return this.root[_global] + if (this.#root === this) { + return this.#global + } + return this.#root.global } // true for packages installed directly in the global node_modules folder @@ -282,21 +284,21 @@ class Node { } get workspaces () { - return this[_workspaces] + return this.#workspaces } set workspaces (workspaces) { // deletes edges if they already exists - if (this[_workspaces]) { - for (const name of this[_workspaces].keys()) { + if (this.#workspaces) { + for (const name of this.#workspaces.keys()) { if (!workspaces.has(name)) { this.edgesOut.get(name).detach() } } } - this[_workspaces] = workspaces - this[_loadWorkspaces]() + this.#workspaces = workspaces + this.#loadWorkspaces() this[_loadDeps]() } @@ -367,7 +369,7 @@ class Node { pkg = {} } this[_package] = pkg - this[_loadWorkspaces]() + this.#loadWorkspaces() this[_loadDeps]() // do a hard reload, since the dependents may now be valid or invalid // as a result of the package change. @@ -569,12 +571,12 @@ class Node { // this allows us to do new Node({...}) and then set the root later. // just make the assignment so we don't lose it, and move on. if (!this.path || !root.realpath || !root.path) { - this[_root] = root + this.#root = root return } // temporarily become a root node - this[_root] = this + this.#root = this // break all linksIn, we're going to re-set them if needed later for (const link of this.linksIn) { @@ -618,7 +620,7 @@ class Node { current.root = null } - this[_root] = root + this.#root = root // set this.location and add to inventory this[_refreshLocation]() @@ -684,22 +686,22 @@ class Node { // the node at nm/a, which might have the root node as a fsParent. // we can't rely on the public setter here, because it calls into // this function to set up these references! - const nmloc = `${this.location}${this.location ? '/' : ''}node_modules/` - const isChild = n => n.location === nmloc + n.name // check dirname so that /foo isn't treated as the fsparent of /foo-bar - const isFsChild = n => { - return dirname(n.path).startsWith(this.path) && - n !== this && - !n.parent && - (!n.fsParent || - n.fsParent === this || - dirname(this.path).startsWith(n.fsParent.path)) - } - const isKid = n => isChild(n) || isFsChild(n) - + const nmloc = `${this.location}${this.location ? '/' : ''}node_modules/` // only walk top nodes, since anything else already has a parent. for (const child of root.tops) { - if (!isKid(child)) { + const isChild = child.location === nmloc + child.name + const isFsChild = + dirname(child.path).startsWith(this.path) && + child !== this && + !child.parent && + ( + !child.fsParent || + child.fsParent === this || + dirname(this.path).startsWith(child.fsParent.path) + ) + + if (!isChild && !isFsChild) { continue } @@ -712,7 +714,7 @@ class Node { child.fsParent.fsChildren.delete(child) } child[_fsParent] = null - if (isChild(child)) { + if (isChild) { this.children.set(child.name, child) child[_parent] = this root.tops.delete(child) @@ -823,19 +825,21 @@ class Node { } // tree should always be valid upon root setter completion. treeCheck(this) - treeCheck(root) + if (this !== root) { + treeCheck(root) + } } get root () { - return this[_root] || this + return this.#root || this } - [_loadWorkspaces] () { - if (!this[_workspaces]) { + #loadWorkspaces () { + if (!this.#workspaces) { return } - for (const [name, path] of this[_workspaces].entries()) { + for (const [name, path] of this.#workspaces.entries()) { new Edge({ from: this, name, spec: `file:${path.replace(/#/g, '%23')}`, type: 'workspace' }) } } @@ -851,23 +855,24 @@ class Node { // but don't have a 'path' field, only a 'realpath', because we // don't know their canonical location. We don't need their devDeps. const pd = this.package.peerDependencies + const ad = this.package.acceptDependencies || {} if (pd && typeof pd === 'object' && !this.legacyPeerDeps) { const pm = this.package.peerDependenciesMeta || {} const peerDependencies = {} const peerOptional = {} for (const [name, dep] of Object.entries(pd)) { - if (pm[name] && pm[name].optional) { + if (pm[name]?.optional) { peerOptional[name] = dep } else { peerDependencies[name] = dep } } - this[_loadDepType](peerDependencies, 'peer') - this[_loadDepType](peerOptional, 'peerOptional') + this.#loadDepType(peerDependencies, 'peer', ad) + this.#loadDepType(peerOptional, 'peerOptional', ad) } - this[_loadDepType](this.package.dependencies, 'prod') - this[_loadDepType](this.package.optionalDependencies, 'optional') + this.#loadDepType(this.package.dependencies, 'prod', ad) + this.#loadDepType(this.package.optionalDependencies, 'optional', ad) const { globalTop, isTop, path, sourceReference } = this const { @@ -878,12 +883,11 @@ class Node { const thisDev = isTop && !globalTop && path const srcDev = !sourceReference || srcTop && !srcGlobalTop && srcPath if (thisDev && srcDev) { - this[_loadDepType](this.package.devDependencies, 'dev') + this.#loadDepType(this.package.devDependencies, 'dev', ad) } } - [_loadDepType] (deps, type) { - const ad = this.package.acceptDependencies || {} + #loadDepType (deps, type, ad) { // Because of the order in which _loadDeps runs, we always want to // prioritize a new edge over an existing one for (const [name, spec] of Object.entries(deps || {})) { @@ -895,14 +899,8 @@ class Node { } get fsParent () { - const parent = this[_fsParent] - /* istanbul ignore next - should be impossible */ - debug(() => { - if (parent === this) { - throw new Error('node set to its own fsParent') - } - }) - return parent + // in debug setter prevents fsParent from being this + return this[_fsParent] } set fsParent (fsParent) { @@ -997,7 +995,7 @@ class Node { // root dependency brings peer deps along with it. In that case, we // will go ahead and create the invalid state, and then try to resolve // it with more tree construction, because it's a user request. - canReplaceWith (node, ignorePeers = []) { + canReplaceWith (node, ignorePeers) { if (node.name !== this.name) { return false } @@ -1010,7 +1008,6 @@ class Node { if (node.overrides !== this.overrides) { return false } - ignorePeers = new Set(ignorePeers) // gather up all the deps of this node and that are only depended @@ -1022,11 +1019,10 @@ class Node { // when replacing peer sets, we need to be able to replace the entire // peer group, which means we ignore incoming edges from other peers // within the replacement set. - const ignored = !this.isTop && + if (!this.isTop && edge.from.parent === this.parent && edge.peer && - ignorePeers.has(edge.from.name) - if (ignored) { + ignorePeers.has(edge.from.name)) { continue } @@ -1156,9 +1152,7 @@ class Node { // something case-insensitively, so merely setting name and path won't // have the desired effect. just set the path so it'll collide in the // parent's children map, and leave it at that. - const nameMatch = node.parent && - node.parent.children.get(this.name) === node - if (nameMatch) { + if (node.parent?.children.get(this.name) === node) { this.path = resolve(node.parent.path, 'node_modules', this.name) } else { this.path = node.path @@ -1193,14 +1187,8 @@ class Node { } get parent () { - const parent = this[_parent] - /* istanbul ignore next - should be impossible */ - debug(() => { - if (parent === this) { - throw new Error('node set to its own parent') - } - }) - return parent + // setter prevents _parent from being this + return this[_parent] } // This setter keeps everything in order when we move a node from @@ -1405,7 +1393,10 @@ class Node { } get depth () { - return this.isTop ? 0 : this.parent.depth + 1 + if (this.isTop) { + return 0 + } + return this.parent.depth + 1 } get isTop () { @@ -1413,7 +1404,10 @@ class Node { } get top () { - return this.isTop ? this : this.parent.top + if (this.isTop) { + return this + } + return this.parent.top } get isFsTop () { @@ -1421,7 +1415,10 @@ class Node { } get fsTop () { - return this.isFsTop ? this : this.fsParent.fsTop + if (this.isFsTop) { + return this + } + return this.fsParent.fsTop } get resolveParent () { diff --git a/workspaces/arborist/test/edge.js b/workspaces/arborist/test/edge.js index f45cb87917702..ab08357ece359 100644 --- a/workspaces/arborist/test/edge.js +++ b/workspaces/arborist/test/edge.js @@ -228,6 +228,16 @@ t.ok(new Edge({ }).satisfiedBy(c), 'c@2 satisfies spec:1.x, accept:2.x') reset(a) +t.equal( + (new Edge({ + from: a, + type: 'prod', + name: 'c', + spec: '1.x', + accept: '2.x', + })).accept, '2.x', '.accept getter works') +reset(a) + t.ok(new Edge({ from: a, type: 'prod',