Current File : //usr/local/lib/node_modules/npm/node_modules/@npmcli/arborist/lib/arborist/load-actual.js
// mix-in implementing the loadActual method
const { relative, dirname, resolve, join, normalize } = require('path')
const rpj = require('read-package-json-fast')
const { promisify } = require('util')
const readdir = promisify(require('readdir-scoped-modules'))
const walkUp = require('walk-up-path')
const ancestorPath = require('common-ancestor-path')
const treeCheck = require('../tree-check.js')
const Shrinkwrap = require('../shrinkwrap.js')
const calcDepFlags = require('../calc-dep-flags.js')
const Node = require('../node.js')
const Link = require('../link.js')
const realpath = require('../realpath.js')
const _loadFSNode = Symbol('loadFSNode')
const _newNode = Symbol('newNode')
const _newLink = Symbol('newLink')
const _loadFSTree = Symbol('loadFSTree')
const _loadFSChildren = Symbol('loadFSChildren')
const _findMissingEdges = Symbol('findMissingEdges')
const _findFSParents = Symbol('findFSParents')
const _resetDepFlags = Symbol('resetDepFlags')
const _actualTreeLoaded = Symbol('actualTreeLoaded')
const _rpcache = Symbol.for('realpathCache')
const _stcache = Symbol.for('statCache')
const _topNodes = Symbol('linkTargets')
const _cache = Symbol('nodeLoadingCache')
const _loadActual = Symbol('loadActual')
const _loadActualVirtually = Symbol('loadActualVirtually')
const _loadActualActually = Symbol('loadActualActually')
const _loadWorkspaces = Symbol.for('loadWorkspaces')
const _loadWorkspaceTargets = Symbol('loadWorkspaceTargets')
const _actualTreePromise = Symbol('actualTreePromise')
const _actualTree = Symbol('actualTree')
const _transplant = Symbol('transplant')
const _transplantFilter = Symbol('transplantFilter')
const _filter = Symbol('filter')
const _global = Symbol.for('global')
const _changePath = Symbol.for('_changePath')
module.exports = cls => class ActualLoader extends cls {
constructor (options) {
super(options)
this[_global] = !!options.global
// the tree of nodes on disk
this.actualTree = options.actualTree
// ensure when walking the tree that we don't call loadTree on the
// same actual node more than one time.
this[_actualTreeLoaded] = new Set()
// caches for cached realpath calls
const cwd = process.cwd()
// assume that the cwd is real enough for our purposes
this[_rpcache] = new Map([[cwd, cwd]])
this[_stcache] = new Map()
// cache of nodes when loading the actualTree, so that we avoid
// loaded the same node multiple times when symlinks attack.
this[_cache] = new Map()
// cache of link targets for setting fsParent links
// We don't do fsParent as a magic getter/setter, because
// it'd be too costly to keep up to date along the walk.
// And, we know that it can ONLY be relevant when the node
// is a target of a link, otherwise it'd be in a node_modules
// folder, so take advantage of that to limit the scans later.
this[_topNodes] = new Set()
}
[_resetDepFlags] (tree, root) {
// reset all deps to extraneous prior to recalc
if (!root) {
for (const node of tree.inventory.values()) {
node.extraneous = true
}
}
// only reset root flags if we're not re-rooting,
// otherwise leave as-is
calcDepFlags(tree, !root)
return tree
}
// public method
async loadActual (options = {}) {
// allow the user to set options on the ctor as well.
// XXX: deprecate separate method options objects.
options = { ...this.options, ...options }
// stash the promise so that we don't ever have more than one
// going at the same time. This is so that buildIdealTree can
// default to the actualTree if no shrinkwrap present, but
// reify() can still call buildIdealTree and loadActual in parallel
// safely.
return this.actualTree ? this.actualTree
: this[_actualTreePromise] ? this[_actualTreePromise]
: this[_actualTreePromise] = this[_loadActual](options)
.then(tree => this[_resetDepFlags](tree, options.root))
.then(tree => this.actualTree = treeCheck(tree))
}
async [_loadActual] (options) {
// mostly realpath to throw if the root doesn't exist
const {
global = false,
filter = () => true,
root = null,
transplantFilter = () => true,
ignoreMissing = false,
} = options
this[_filter] = filter
this[_transplantFilter] = transplantFilter
if (global) {
const real = await realpath(this.path, this[_rpcache], this[_stcache])
const newNodeOrLink = this.path === real ? _newNode : _newLink
this[_actualTree] = await this[newNodeOrLink]({
path: this.path,
realpath: real,
pkg: {},
global,
loadOverrides: true,
})
return this[_loadActualActually]({ root, ignoreMissing, global })
}
// not in global mode, hidden lockfile is allowed, load root pkg too
this[_actualTree] = await this[_loadFSNode]({
path: this.path,
real: await realpath(this.path, this[_rpcache], this[_stcache]),
loadOverrides: true,
})
this[_actualTree].assertRootOverrides()
// Note: hidden lockfile will be rejected if it's not the latest thing
// in the folder, or if any of the entries in the hidden lockfile are
// missing.
const meta = await Shrinkwrap.load({
path: this[_actualTree].path,
hiddenLockfile: true,
resolveOptions: this.options,
})
if (meta.loadedFromDisk) {
this[_actualTree].meta = meta
return this[_loadActualVirtually]({ root })
} else {
const meta = await Shrinkwrap.load({
path: this[_actualTree].path,
lockfileVersion: this.options.lockfileVersion,
resolveOptions: this.options,
})
this[_actualTree].meta = meta
return this[_loadActualActually]({ root, ignoreMissing })
}
}
async [_loadActualVirtually] ({ root }) {
// have to load on a new Arborist object, so we don't assign
// the virtualTree on this one! Also, the weird reference is because
// we can't easily get a ref to Arborist in this module, without
// creating a circular reference, since this class is a mixin used
// to build up the Arborist class itself.
await new this.constructor({ ...this.options }).loadVirtual({
root: this[_actualTree],
})
await this[_loadWorkspaces](this[_actualTree])
this[_transplant](root)
return this[_actualTree]
}
async [_loadActualActually] ({ root, ignoreMissing, global }) {
await this[_loadFSTree](this[_actualTree])
await this[_loadWorkspaces](this[_actualTree])
await this[_loadWorkspaceTargets](this[_actualTree])
if (!ignoreMissing) {
await this[_findMissingEdges]()
}
this[_findFSParents]()
this[_transplant](root)
if (global) {
// need to depend on the children, or else all of them
// will end up being flagged as extraneous, since the
// global root isn't a "real" project
const tree = this[_actualTree]
const actualRoot = tree.isLink ? tree.target : tree
const { dependencies = {} } = actualRoot.package
for (const [name, kid] of actualRoot.children.entries()) {
const def = kid.isLink ? `file:${kid.realpath}` : '*'
dependencies[name] = dependencies[name] || def
}
actualRoot.package = { ...actualRoot.package, dependencies }
}
return this[_actualTree]
}
// if there are workspace targets without Link nodes created, load
// the targets, so that we know what they are.
async [_loadWorkspaceTargets] (tree) {
if (!tree.workspaces || !tree.workspaces.size) {
return
}
const promises = []
for (const path of tree.workspaces.values()) {
if (!this[_cache].has(path)) {
// workspace overrides use the root overrides
const p = this[_loadFSNode]({ path, root: this[_actualTree], useRootOverrides: true })
.then(node => this[_loadFSTree](node))
promises.push(p)
}
}
await Promise.all(promises)
}
[_transplant] (root) {
if (!root || root === this[_actualTree]) {
return
}
this[_actualTree][_changePath](root.path)
for (const node of this[_actualTree].children.values()) {
if (!this[_transplantFilter](node)) {
node.root = null
}
}
root.replace(this[_actualTree])
for (const node of this[_actualTree].fsChildren) {
node.root = this[_transplantFilter](node) ? root : null
}
this[_actualTree] = root
}
[_loadFSNode] ({ path, parent, real, root, loadOverrides, useRootOverrides }) {
if (!real) {
return realpath(path, this[_rpcache], this[_stcache])
.then(
real => this[_loadFSNode]({
path,
parent,
real,
root,
loadOverrides,
useRootOverrides,
}),
// if realpath fails, just provide a dummy error node
error => new Node({
error,
path,
realpath: path,
parent,
root,
loadOverrides,
})
)
}
// cache temporarily holds a promise placeholder so we don't try to create
// the same node multiple times. this is rare to encounter, given the
// aggressive caching on realpath and lstat calls, but it's possible that
// it's already loaded as a tree top, and then gets its parent loaded
// later, if a symlink points deeper in the tree.
const cached = this[_cache].get(path)
if (cached && !cached.dummy) {
return Promise.resolve(cached).then(node => {
node.parent = parent
return node
})
}
const p = rpj(join(real, 'package.json'))
// soldier on if read-package-json raises an error
.then(pkg => [pkg, null], error => [null, error])
.then(([pkg, error]) => {
return this[normalize(path) === real ? _newNode : _newLink]({
installLinks: this.installLinks,
legacyPeerDeps: this.legacyPeerDeps,
path,
realpath: real,
pkg,
error,
parent,
root,
loadOverrides,
...(useRootOverrides && root.overrides
? { overrides: root.overrides.getNodeRule({ name: pkg.name, version: pkg.version }) }
: {}),
})
})
.then(node => {
this[_cache].set(path, node)
return node
})
this[_cache].set(path, p)
return p
}
// this is the way it is to expose a timing issue which is difficult to
// test otherwise. The creation of a Node may take slightly longer than
// the creation of a Link that targets it. If the Node has _begun_ its
// creation phase (and put a Promise in the cache) then the Link will
// get a Promise as its cachedTarget instead of an actual Node object.
// This is not a problem, because it gets resolved prior to returning
// the tree or attempting to load children. However, it IS remarkably
// difficult to get to happen in a test environment to verify reliably.
// Hence this kludge.
[_newNode] (options) {
// check it for an fsParent if it's a tree top. there's a decent chance
// it'll get parented later, making the fsParent scan a no-op, but better
// safe than sorry, since it's cheap.
const { parent, realpath } = options
if (!parent) {
this[_topNodes].add(realpath)
}
return process.env._TEST_ARBORIST_SLOW_LINK_TARGET_ === '1'
? new Promise(res => setTimeout(() => res(new Node(options)), 100))
: new Node(options)
}
[_newLink] (options) {
const { realpath } = options
this[_topNodes].add(realpath)
const target = this[_cache].get(realpath)
const link = new Link({ ...options, target })
if (!target) {
this[_cache].set(realpath, link.target)
// if a link target points at a node outside of the root tree's
// node_modules hierarchy, then load that node as well.
return this[_loadFSTree](link.target).then(() => link)
} else if (target.then) {
target.then(node => link.target = node)
}
return link
}
[_loadFSTree] (node) {
const did = this[_actualTreeLoaded]
node = node.target
// if a Link target has started, but not completed, then
// a Promise will be in the cache to indicate this.
if (node.then) {
return node.then(node => this[_loadFSTree](node))
}
// impossible except in pathological ELOOP cases
/* istanbul ignore if */
if (did.has(node.realpath)) {
return Promise.resolve(node)
}
did.add(node.realpath)
return this[_loadFSChildren](node)
.then(() => Promise.all(
[...node.children.entries()]
.filter(([name, kid]) => !did.has(kid.realpath))
.map(([name, kid]) => this[_loadFSTree](kid))))
}
// create child nodes for all the entries in node_modules
// and attach them to the node as a parent
[_loadFSChildren] (node) {
const nm = resolve(node.realpath, 'node_modules')
return readdir(nm).then(kids => {
return Promise.all(
// ignore . dirs and retired scoped package folders
kids.filter(kid => !/^(@[^/]+\/)?\./.test(kid))
.filter(kid => this[_filter](node, kid))
.map(kid => this[_loadFSNode]({
parent: node,
path: resolve(nm, kid),
})))
},
// error in the readdir is not fatal, just means no kids
() => {})
}
async [_findMissingEdges] () {
// try to resolve any missing edges by walking up the directory tree,
// checking for the package in each node_modules folder. stop at the
// root directory.
// The tricky move here is that we load a "dummy" node for the folder
// containing the node_modules folder, so that it can be assigned as
// the fsParent. It's a bad idea to *actually* load that full node,
// because people sometimes develop in ~/projects/node_modules/...
// so we'd end up loading a massive tree with lots of unrelated junk.
const nmContents = new Map()
const tree = this[_actualTree]
for (const node of tree.inventory.values()) {
const ancestor = ancestorPath(node.realpath, this.path)
const depPromises = []
for (const [name, edge] of node.edgesOut.entries()) {
const notMissing = !edge.missing &&
!(edge.to && (edge.to.dummy || edge.to.parent !== node))
if (notMissing) {
continue
}
// start the walk from the dirname, because we would have found
// the dep in the loadFSTree step already if it was local.
for (const p of walkUp(dirname(node.realpath))) {
// only walk as far as the nearest ancestor
// this keeps us from going into completely unrelated
// places when a project is just missing something, but
// allows for finding the transitive deps of link targets.
// ie, if it has to go up and back out to get to the path
// from the nearest common ancestor, we've gone too far.
if (ancestor && /^\.\.(?:[\\/]|$)/.test(relative(ancestor, p))) {
break
}
const entries = nmContents.get(p) ||
await readdir(p + '/node_modules').catch(() => [])
nmContents.set(p, entries)
if (!entries.includes(name)) {
continue
}
const d = this[_cache].has(p) ? await this[_cache].get(p)
: new Node({ path: p, root: node.root, dummy: true })
this[_cache].set(p, d)
if (d.dummy) {
// it's a placeholder, so likely would not have loaded this dep,
// unless another dep in the tree also needs it.
const depPath = normalize(`${p}/node_modules/${name}`)
const cached = this[_cache].get(depPath)
if (!cached || cached.dummy) {
depPromises.push(this[_loadFSNode]({
path: depPath,
root: node.root,
parent: d,
}).then(node => this[_loadFSTree](node)))
}
}
break
}
}
await Promise.all(depPromises)
}
}
// try to find a node that is the parent in a fs tree sense, but not a
// node_modules tree sense, of any link targets. this allows us to
// resolve deps that node will find, but a legacy npm view of the
// world would not have noticed.
[_findFSParents] () {
for (const path of this[_topNodes]) {
const node = this[_cache].get(path)
if (node && !node.parent && !node.fsParent) {
for (const p of walkUp(dirname(path))) {
if (this[_cache].has(p)) {
node.fsParent = this[_cache].get(p)
break
}
}
}
}
}
}
Mini Shell