/
opt
/
alt
/
alt-nodejs18
/
root
/
lib
/
node_modules
/
npm
/
node_modules
/
@npmcli
/
arborist
/
lib
/
arborist
/
Upload Filee
HOME
// mixin implementing the reify method const onExit = require('../signal-handling.js') const pacote = require('pacote') const AuditReport = require('../audit-report.js') const { subset, intersects } = require('semver') const npa = require('npm-package-arg') const semver = require('semver') const debug = require('../debug.js') const { walkUp } = require('walk-up-path') const { log, time } = require('proc-log') const hgi = require('hosted-git-info') const rpj = require('read-package-json-fast') const { dirname, resolve, relative, join } = require('path') const { depth: dfwalk } = require('treeverse') const { lstat, mkdir, rm, symlink, } = require('fs/promises') const { moveFile } = require('@npmcli/fs') const PackageJson = require('@npmcli/package-json') const packageContents = require('@npmcli/installed-package-contents') const runScript = require('@npmcli/run-script') const { checkEngine, checkPlatform } = require('npm-install-checks') const treeCheck = require('../tree-check.js') const relpath = require('../relpath.js') const Diff = require('../diff.js') const retirePath = require('../retire-path.js') const promiseAllRejectLate = require('promise-all-reject-late') const { callLimit: promiseCallLimit } = require('promise-call-limit') const optionalSet = require('../optional-set.js') const calcDepFlags = require('../calc-dep-flags.js') const { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js') const Shrinkwrap = require('../shrinkwrap.js') const { defaultLockfileVersion } = Shrinkwrap // Part of steps (steps need refactoring before we can do anything about these) const _retireShallowNodes = Symbol.for('retireShallowNodes') const _loadBundlesAndUpdateTrees = Symbol.for('loadBundlesAndUpdateTrees') const _submitQuickAudit = Symbol('submitQuickAudit') const _addOmitsToTrashList = Symbol('addOmitsToTrashList') const _unpackNewModules = Symbol.for('unpackNewModules') const _build = Symbol.for('build') // shared by rebuild mixin const _trashList = Symbol.for('trashList') const _handleOptionalFailure = Symbol.for('handleOptionalFailure') const _loadTrees = Symbol.for('loadTrees') // defined by rebuild mixin const _checkBins = Symbol.for('checkBins') // shared symbols for swapping out when testing // TODO tests should not be this deep into internals const _diffTrees = Symbol.for('diffTrees') const _createSparseTree = Symbol.for('createSparseTree') const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees') const _reifyNode = Symbol.for('reifyNode') const _updateAll = Symbol.for('updateAll') const _updateNames = Symbol.for('updateNames') const _moveContents = Symbol.for('moveContents') const _moveBackRetiredUnchanged = Symbol.for('moveBackRetiredUnchanged') const _removeTrash = Symbol.for('removeTrash') const _renamePath = Symbol.for('renamePath') const _rollbackRetireShallowNodes = Symbol.for('rollbackRetireShallowNodes') const _rollbackCreateSparseTree = Symbol.for('rollbackCreateSparseTree') const _rollbackMoveBackRetiredUnchanged = Symbol.for('rollbackMoveBackRetiredUnchanged') const _saveIdealTree = Symbol.for('saveIdealTree') const _reifyPackages = Symbol.for('reifyPackages') // defined by build-ideal-tree mixin const _resolvedAdd = Symbol.for('resolvedAdd') const _usePackageLock = Symbol.for('usePackageLock') // used by build-ideal-tree mixin const _addNodeToTrashList = Symbol.for('addNodeToTrashList') const _createIsolatedTree = Symbol.for('createIsolatedTree') module.exports = cls => class Reifier extends cls { #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't #bundleUnpacked = new Set() // the nodes we unpack to read their bundles #dryRun #nmValidated = new Set() #omitDev #omitPeer #omitOptional #retiredPaths = {} #retiredUnchanged = {} #savePrefix #shrinkwrapInflated = new Set() #sparseTreeDirs = new Set() #sparseTreeRoots = new Set() constructor (options) { super(options) this[_trashList] = new Set() } // public method async reify (options = {}) { const linked = (options.installStrategy || this.options.installStrategy) === 'linked' if (this.options.packageLockOnly && this.options.global) { const er = new Error('cannot generate lockfile for global packages') er.code = 'ESHRINKWRAPGLOBAL' throw er } const omit = new Set(options.omit || []) this.#omitDev = omit.has('dev') this.#omitOptional = omit.has('optional') this.#omitPeer = omit.has('peer') // start tracker block this.addTracker('reify') const timeEnd = time.start('reify') // don't create missing dirs on dry runs if (!this.options.packageLockOnly && !this.options.dryRun) { // we do NOT want to set ownership on this folder, especially // recursively, because it can have other side effects to do that // in a project directory. We just want to make it if it's missing. await mkdir(resolve(this.path), { recursive: true }) // do not allow the top-level node_modules to be a symlink await this.#validateNodeModules(resolve(this.path, 'node_modules')) } await this[_loadTrees](options) const oldTree = this.idealTree if (linked) { // swap out the tree with the isolated tree // this is currently technical debt which will be resolved in a refactor // of Node/Link trees log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.') this.idealTree = await this[_createIsolatedTree]() } await this[_diffTrees]() await this[_reifyPackages]() if (linked) { // swap back in the idealTree // so that the lockfile is preserved this.idealTree = oldTree } await this[_saveIdealTree](options) // clean up any trash that is still in the tree for (const path of this[_trashList]) { const loc = relpath(this.idealTree.realpath, path) const node = this.idealTree.inventory.get(loc) if (node && node.root === this.idealTree) { node.parent = null } } // if we filtered to only certain nodes, then anything ELSE needs // to be untouched in the resulting actual tree, even if it differs // in the idealTree. Copy over anything that was in the actual and // was not changed, delete anything in the ideal and not actual. // Then we move the entire idealTree over to this.actualTree, and // save the hidden lockfile. if (this.diff && this.diff.filterSet.size) { const reroot = new Set() const { filterSet } = this.diff const seen = new Set() for (const [loc, ideal] of this.idealTree.inventory.entries()) { seen.add(loc) // if it's an ideal node from the filter set, then skip it // because we already made whatever changes were necessary if (filterSet.has(ideal)) { continue } // otherwise, if it's not in the actualTree, then it's not a thing // that we actually added. And if it IS in the actualTree, then // it's something that we left untouched, so we need to record // that. const actual = this.actualTree.inventory.get(loc) if (!actual) { ideal.root = null } else { if ([...actual.linksIn].some(link => filterSet.has(link))) { seen.add(actual.location) continue } const { realpath, isLink } = actual if (isLink && ideal.isLink && ideal.realpath === realpath) { continue } else { reroot.add(actual) } } } // now find any actual nodes that may not be present in the ideal // tree, but were left behind by virtue of not being in the filter for (const [loc, actual] of this.actualTree.inventory.entries()) { if (seen.has(loc)) { continue } seen.add(loc) // we know that this is something that ISN'T in the idealTree, // or else we will have addressed it in the previous loop. // If it's in the filterSet, that means we intentionally removed // it, so nothing to do here. if (filterSet.has(actual)) { continue } reroot.add(actual) } // go through the rerooted actual nodes, and move them over. for (const actual of reroot) { actual.root = this.idealTree } // prune out any tops that lack a linkIn, they are no longer relevant. for (const top of this.idealTree.tops) { if (top.linksIn.size === 0) { top.root = null } } // need to calculate dep flags, since nodes may have been marked // as extraneous or otherwise incorrect during transit. calcDepFlags(this.idealTree) } // save the ideal's meta as a hidden lockfile after we actualize it this.idealTree.meta.filename = this.idealTree.realpath + '/node_modules/.package-lock.json' this.idealTree.meta.hiddenLockfile = true this.idealTree.meta.lockfileVersion = defaultLockfileVersion this.actualTree = this.idealTree this.idealTree = null if (!this.options.global) { await this.actualTree.meta.save() const ignoreScripts = !!this.options.ignoreScripts // if we aren't doing a dry run or ignoring scripts and we actually made changes to the dep // tree, then run the dependencies scripts if (!this.options.dryRun && !ignoreScripts && this.diff && this.diff.children.length) { const { path, package: pkg } = this.actualTree.target const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe' const { scripts = {} } = pkg for (const event of ['predependencies', 'dependencies', 'postdependencies']) { if (Object.prototype.hasOwnProperty.call(scripts, event)) { log.info('run', pkg._id, event, scripts[event]) await time.start(`reify:run:${event}`, () => runScript({ event, path, pkg, stdio, scriptShell: this.options.scriptShell, })) } } } } // This is a very bad pattern and I can't wait to stop doing it this.auditReport = await this.auditReport this.finishTracker('reify') timeEnd() return treeCheck(this.actualTree) } async [_reifyPackages] () { // we don't submit the audit report or write to disk on dry runs if (this.options.dryRun) { return } if (this.options.packageLockOnly) { // we already have the complete tree, so just audit it now, // and that's all we have to do here. return this[_submitQuickAudit]() } // ok, we're about to start touching the fs. need to roll back // if we get an early termination. let reifyTerminated = null const removeHandler = onExit(({ signal }) => { // only call once. if signal hits twice, we just terminate removeHandler() reifyTerminated = Object.assign(new Error('process terminated'), { signal, }) return false }) // [rollbackfn, [...actions]] // after each step, if the process was terminated, execute the rollback // note that each rollback *also* calls the previous one when it's // finished, and then the first one throws the error, so we only need // a new rollback step when we have a new thing that must be done to // revert the install. const steps = [ [_rollbackRetireShallowNodes, [ _retireShallowNodes, ]], [_rollbackCreateSparseTree, [ _createSparseTree, _addOmitsToTrashList, _loadShrinkwrapsAndUpdateTrees, _loadBundlesAndUpdateTrees, _submitQuickAudit, _unpackNewModules, ]], [_rollbackMoveBackRetiredUnchanged, [ _moveBackRetiredUnchanged, _build, ]], ] for (const [rollback, actions] of steps) { for (const action of actions) { try { await this[action]() if (reifyTerminated) { throw reifyTerminated } } catch (er) { // TODO rollbacks shouldn't be relied on to throw err await this[rollback](er) /* istanbul ignore next - rollback throws, should never hit this */ throw er } } } // no rollback for this one, just exit with the error, since the // install completed and can't be safely recovered at this point. await this[_removeTrash]() if (reifyTerminated) { throw reifyTerminated } // done modifying the file system, no need to keep listening for sigs removeHandler() } // when doing a local install, we load everything and figure it all out. // when doing a global install, we *only* care about the explicit requests. [_loadTrees] (options) { const timeEnd = time.start('reify:loadTrees') const bitOpt = { ...options, complete: this.options.packageLockOnly || this.options.dryRun, } // if we're only writing a package lock, then it doesn't matter what's here if (this.options.packageLockOnly) { return this.buildIdealTree(bitOpt).then(timeEnd) } const actualOpt = this.options.global ? { ignoreMissing: true, global: true, filter: (node, kid) => { // if it's not the project root, and we have no explicit requests, // then we're already into a nested dep, so we keep it if (this.explicitRequests.size === 0 || !node.isProjectRoot) { return true } // if we added it as an edgeOut, then we want it if (this.idealTree.edgesOut.has(kid)) { return true } // if it's an explicit request, then we want it const hasExplicit = [...this.explicitRequests] .some(edge => edge.name === kid) if (hasExplicit) { return true } // ignore the rest of the global install folder return false }, } : { ignoreMissing: true } if (!this.options.global) { return Promise.all([ this.loadActual(actualOpt), this.buildIdealTree(bitOpt), ]).then(timeEnd) } // the global install space tends to have a lot of stuff in it. don't // load all of it, just what we care about. we won't be saving a // hidden lockfile in there anyway. Note that we have to load ideal // BEFORE loading actual, so that the actualOpt can use the // explicitRequests which is set during buildIdealTree return this.buildIdealTree(bitOpt) .then(() => this.loadActual(actualOpt)) .then(timeEnd) } [_diffTrees] () { if (this.options.packageLockOnly) { return } const timeEnd = time.start('reify:diffTrees') // XXX if we have an existing diff already, there should be a way // to just invalidate the parts that changed, but avoid walking the // whole tree again. const includeWorkspaces = this.options.workspacesEnabled const includeRootDeps = !includeWorkspaces || this.options.includeWorkspaceRoot && this.options.workspaces.length > 0 const filterNodes = [] if (this.options.global && this.explicitRequests.size) { const idealTree = this.idealTree.target const actualTree = this.actualTree.target // we ONLY are allowed to make changes in the global top-level // children where there's an explicit request. for (const { name } of this.explicitRequests) { const ideal = idealTree.children.get(name) if (ideal) { filterNodes.push(ideal) } const actual = actualTree.children.get(name) if (actual) { filterNodes.push(actual) } } } else { if (includeWorkspaces) { // add all ws nodes to filterNodes for (const ws of this.options.workspaces) { const ideal = this.idealTree.children.get(ws) if (ideal) { filterNodes.push(ideal) } const actual = this.actualTree.children.get(ws) if (actual) { filterNodes.push(actual) } } } if (includeRootDeps) { // add all non-workspace nodes to filterNodes for (const tree of [this.idealTree, this.actualTree]) { for (const { type, to } of tree.edgesOut.values()) { if (type !== 'workspace' && to) { filterNodes.push(to) } } } } } // find all the nodes that need to change between the actual // and ideal trees. this.diff = Diff.calculate({ shrinkwrapInflated: this.#shrinkwrapInflated, filterNodes, actual: this.actualTree, ideal: this.idealTree, }) // we don't have to add 'removed' folders to the trashlist, because // they'll be moved aside to a retirement folder, and then the retired // folder will be deleted at the end. This is important when we have // a folder like FOO being "removed" in favor of a folder like "foo", // because if we remove node_modules/FOO on case-insensitive systems, // it will remove the dep that we *want* at node_modules/foo. timeEnd() } // add the node and all its bins to the list of things to be // removed later on in the process. optionally, also mark them // as a retired paths, so that we move them out of the way and // replace them when rolling back on failure. [_addNodeToTrashList] (node, retire = false) { const paths = [node.path, ...node.binPaths] const moves = this.#retiredPaths log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths) for (const path of paths) { if (retire) { const retired = retirePath(path) moves[path] = retired this[_trashList].add(retired) } else { this[_trashList].add(path) } } } // move aside the shallowest nodes in the tree that have to be // changed or removed, so that we can rollback if necessary. [_retireShallowNodes] () { const timeEnd = time.start('reify:retireShallow') const moves = this.#retiredPaths = {} for (const diff of this.diff.children) { if (diff.action === 'CHANGE' || diff.action === 'REMOVE') { // we'll have to clean these up at the end, so add them to the list this[_addNodeToTrashList](diff.actual, true) } } log.silly('reify', 'moves', moves) const movePromises = Object.entries(moves) .map(([from, to]) => this[_renamePath](from, to)) return promiseAllRejectLate(movePromises).then(timeEnd) } [_renamePath] (from, to, didMkdirp = false) { return moveFile(from, to) .catch(er => { // Occasionally an expected bin file might not exist in the package, // or a shim/symlink might have been moved aside. If we've already // handled the most common cause of ENOENT (dir doesn't exist yet), // then just ignore any ENOENT. if (er.code === 'ENOENT') { return didMkdirp ? null : mkdir(dirname(to), { recursive: true }).then(() => this[_renamePath](from, to, true)) } else if (er.code === 'EEXIST') { return rm(to, { recursive: true, force: true }).then(() => moveFile(from, to)) } else { throw er } }) } [_rollbackRetireShallowNodes] (er) { const timeEnd = time.start('reify:rollback:retireShallow') const moves = this.#retiredPaths const movePromises = Object.entries(moves) .map(([from, to]) => this[_renamePath](to, from)) return promiseAllRejectLate(movePromises) // ignore subsequent rollback errors .catch(() => {}) .then(timeEnd) .then(() => { throw er }) } // adding to the trash list will skip reifying, and delete them // if they are currently in the tree and otherwise untouched. [_addOmitsToTrashList] () { if (!this.#omitDev && !this.#omitOptional && !this.#omitPeer) { return } const timeEnd = time.start('reify:trashOmits') for (const node of this.idealTree.inventory.values()) { const { top } = node // if the top is not the root or workspace then we do not want to omit it if (!top.isProjectRoot && !top.isWorkspace) { continue } // if a diff filter has been created, then we do not omit the node if the // top node is not in that set if (this.diff?.filterSet?.size && !this.diff.filterSet.has(top)) { continue } // omit node if the dep type matches any omit flags that were set if ( node.peer && this.#omitPeer || node.dev && this.#omitDev || node.optional && this.#omitOptional || node.devOptional && this.#omitOptional && this.#omitDev ) { this[_addNodeToTrashList](node) } } timeEnd() } [_createSparseTree] () { const timeEnd = time.start('reify:createSparse') // if we call this fn again, we look for the previous list // so that we can avoid making the same directory multiple times const leaves = this.diff.leaves .filter(diff => { return (diff.action === 'ADD' || diff.action === 'CHANGE') && !this.#sparseTreeDirs.has(diff.ideal.path) && !diff.ideal.isLink }) .map(diff => diff.ideal) // we check this in parallel, so guard against multiple attempts to // retire the same path at the same time. const dirsChecked = new Set() return promiseAllRejectLate(leaves.map(async node => { for (const d of walkUp(node.path)) { if (d === node.top.path) { break } if (dirsChecked.has(d)) { continue } dirsChecked.add(d) const st = await lstat(d).catch(() => null) // this can happen if we have a link to a package with a name // that the filesystem treats as if it is the same thing. // would be nice to have conditional istanbul ignores here... /* istanbul ignore next - defense in depth */ if (st && !st.isDirectory()) { const retired = retirePath(d) this.#retiredPaths[d] = retired this[_trashList].add(retired) await this[_renamePath](d, retired) } } this.#sparseTreeDirs.add(node.path) const made = await mkdir(node.path, { recursive: true }) // if the directory already exists, made will be undefined. if that's the case // we don't want to remove it because we aren't the ones who created it so we // omit it from the #sparseTreeRoots if (made) { this.#sparseTreeRoots.add(made) } })).then(timeEnd) } [_rollbackCreateSparseTree] (er) { const timeEnd = time.start('reify:rollback:createSparse') // cut the roots of the sparse tree that were created, not the leaves const roots = this.#sparseTreeRoots // also delete the moves that we retired, so that we can move them back const failures = [] const targets = [...roots, ...Object.keys(this.#retiredPaths)] const unlinks = targets .map(path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er]))) return promiseAllRejectLate(unlinks).then(() => { // eslint-disable-next-line promise/always-return if (failures.length) { log.warn('cleanup', 'Failed to remove some directories', failures) } }) .then(timeEnd) .then(() => this[_rollbackRetireShallowNodes](er)) } // shrinkwrap nodes define their dependency branches with a file, so // we need to unpack them, read that shrinkwrap file, and then update // the tree by calling loadVirtual with the node as the root. [_loadShrinkwrapsAndUpdateTrees] () { const seen = this.#shrinkwrapInflated const shrinkwraps = this.diff.leaves .filter(d => (d.action === 'CHANGE' || d.action === 'ADD' || !d.action) && d.ideal.hasShrinkwrap && !seen.has(d.ideal) && !this[_trashList].has(d.ideal.path)) if (!shrinkwraps.length) { return } const timeEnd = time.start('reify:loadShrinkwraps') const Arborist = this.constructor return promiseAllRejectLate(shrinkwraps.map(diff => { const node = diff.ideal seen.add(node) return diff.action ? this[_reifyNode](node) : node })) .then(nodes => promiseAllRejectLate(nodes.map(node => new Arborist({ ...this.options, path: node.path, }).loadVirtual({ root: node })))) // reload the diff and sparse tree because the ideal tree changed .then(() => this[_diffTrees]()) .then(() => this[_createSparseTree]()) .then(() => this[_addOmitsToTrashList]()) .then(() => this[_loadShrinkwrapsAndUpdateTrees]()) .then(timeEnd) } // create a symlink for Links, extract for Nodes // return the node object, since we usually want that // handle optional dep failures here // If node is in trash list, skip it // If reifying fails, and the node is optional, add it and its optionalSet // to the trash list // Always return the node. [_reifyNode] (node) { if (this[_trashList].has(node.path)) { return node } const timeEnd = time.start(`reifyNode:${node.location}`) this.addTracker('reify', node.name, node.location) const { npmVersion, nodeVersion, cpu, os, libc } = this.options const p = Promise.resolve().then(async () => { // when we reify an optional node, check the engine and platform // first. be sure to ignore the --force and --engine-strict flags, // since we always want to skip any optional packages we can't install. // these checks throwing will result in a rollback and removal // of the mismatches // eslint-disable-next-line promise/always-return if (node.optional) { checkEngine(node.package, npmVersion, nodeVersion, false) checkPlatform(node.package, false, { cpu, os, libc }) } await this[_checkBins](node) await this.#extractOrLink(node) const { _id, deprecated } = node.package // The .catch is in _handleOptionalFailure. Not ideal, this should be cleaned up. // eslint-disable-next-line promise/always-return if (deprecated) { log.warn('deprecated', `${_id}: ${deprecated}`) } }) return this[_handleOptionalFailure](node, p) .then(() => { this.finishTracker('reify', node.name, node.location) timeEnd() return node }) } // do not allow node_modules to be a symlink async #validateNodeModules (nm) { if (this.options.force || this.#nmValidated.has(nm)) { return } const st = await lstat(nm).catch(() => null) if (!st || st.isDirectory()) { this.#nmValidated.add(nm) return } log.warn('reify', 'Removing non-directory', nm) await rm(nm, { recursive: true, force: true }) } async #extractOrLink (node) { const nm = resolve(node.parent.path, 'node_modules') await this.#validateNodeModules(nm) if (!node.isLink) { // in normal cases, node.resolved should *always* be set by now. // however, it is possible when a lockfile is damaged, or very old, // or in some other race condition bugs in npm v6, that a previously // bundled dependency will have just a version, but no resolved value, // and no 'bundled: true' setting. // Do the best with what we have, or else remove it from the tree // entirely, since we can't possibly reify it. let res = null if (node.resolved) { const registryResolved = this.#registryResolved(node.resolved) if (registryResolved) { res = `${node.name}@${registryResolved}` } } else if (node.package.name && node.version) { res = `${node.package.name}@${node.version}` } // no idea what this thing is. remove it from the tree. if (!res) { const warning = 'invalid or damaged lockfile detected\n' + 'please re-try this operation once it completes\n' + 'so that the damage can be corrected, or perform\n' + 'a fresh install with no lockfile if the problem persists.' log.warn('reify', warning) log.verbose('reify', 'unrecognized node in tree', node.path) node.parent = null node.fsParent = null this[_addNodeToTrashList](node) return } await debug(async () => { const st = await lstat(node.path).catch(() => null) if (st && !st.isDirectory()) { debug.log('unpacking into a non-directory', node) throw Object.assign(new Error('ENOTDIR: not a directory'), { code: 'ENOTDIR', path: node.path, }) } }) await pacote.extract(res, node.path, { ...this.options, resolved: node.resolved, integrity: node.integrity, }) // store nodes don't use Node class so node.package doesn't get updated if (node.isInStore) { const pkg = await rpj(join(node.path, 'package.json')) node.package.scripts = pkg.scripts } return } // node.isLink await rm(node.path, { recursive: true, force: true }) // symlink const dir = dirname(node.path) const target = node.realpath const rel = relative(dir, target) await mkdir(dir, { recursive: true }) return symlink(rel, node.path, 'junction') } // if the node is optional, then the failure of the promise is nonfatal // just add it and its optional set to the trash list. [_handleOptionalFailure] (node, p) { return (node.optional ? p.catch(() => { const set = optionalSet(node) for (node of set) { log.verbose('reify', 'failed optional dependency', node.path) this[_addNodeToTrashList](node) } }) : p).then(() => node) } #registryResolved (resolved) { // the default registry url is a magic value meaning "the currently // configured registry". // `resolved` must never be falsey. // // XXX: use a magic string that isn't also a valid value, like // ${REGISTRY} or something. This has to be threaded through the // Shrinkwrap and Node classes carefully, so for now, just treat // the default reg as the magical animal that it has been. const resolvedURL = hgi.parseUrl(resolved) if (!resolvedURL) { // if we could not parse the url at all then returning nothing // here means it will get removed from the tree in the next step return } if ((this.options.replaceRegistryHost === resolvedURL.hostname) || this.options.replaceRegistryHost === 'always') { // this.registry always has a trailing slash return `${this.registry.slice(0, -1)}${resolvedURL.pathname}${resolvedURL.searchParams}` } return resolved } // bundles are *sort of* like shrinkwraps, in that the branch is defined // by the contents of the package. however, in their case, rather than // shipping a virtual tree that must be reified, they ship an entire // reified actual tree that must be unpacked and not modified. [_loadBundlesAndUpdateTrees] (depth = 0, bundlesByDepth) { let maxBundleDepth if (!bundlesByDepth) { bundlesByDepth = new Map() maxBundleDepth = -1 dfwalk({ tree: this.diff, visit: diff => { const node = diff.ideal if (!node) { return } if (node.isProjectRoot) { return } const { bundleDependencies } = node.package if (bundleDependencies && bundleDependencies.length) { maxBundleDepth = Math.max(maxBundleDepth, node.depth) if (!bundlesByDepth.has(node.depth)) { bundlesByDepth.set(node.depth, [node]) } else { bundlesByDepth.get(node.depth).push(node) } } }, getChildren: diff => diff.children, }) bundlesByDepth.set('maxBundleDepth', maxBundleDepth) } else { maxBundleDepth = bundlesByDepth.get('maxBundleDepth') } if (depth === 0) { time.start('reify:loadBundles') } if (depth > maxBundleDepth) { // if we did something, then prune the tree and update the diffs if (maxBundleDepth !== -1) { this.#pruneBundledMetadeps(bundlesByDepth) this[_diffTrees]() } time.end('reify:loadBundles') return } // skip any that have since been removed from the tree, eg by a // shallower bundle overwriting them with a bundled meta-dep. const set = (bundlesByDepth.get(depth) || []) .filter(node => node.root === this.idealTree && node.target !== node.root && !this[_trashList].has(node.path)) if (!set.length) { return this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth) } // extract all the nodes with bundles return promiseCallLimit(set.map(node => { return () => { this.#bundleUnpacked.add(node) return this[_reifyNode](node) } }), { rejectLate: true }) // then load their unpacked children and move into the ideal tree .then(nodes => promiseAllRejectLate(nodes.map(async node => { const arb = new this.constructor({ ...this.options, path: node.path, }) const notTransplanted = new Set(node.children.keys()) await arb.loadActual({ root: node, // don't transplant any sparse folders we created // loadActual will set node.package to {} for empty directories // if by chance there are some empty folders in the node_modules // tree for some other reason, then ok, ignore those too. transplantFilter: node => { if (node.package._id) { // it's actually in the bundle if it gets transplanted notTransplanted.delete(node.name) return true } else { return false } }, }) for (const name of notTransplanted) { this.#bundleMissing.add(node.children.get(name)) } }))) // move onto the next level of bundled items .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)) } // https://github.com/npm/cli/issues/1597#issuecomment-667639545 #pruneBundledMetadeps (bundlesByDepth) { const bundleShadowed = new Set() // Example dep graph: // root -> (a, c) // a -> BUNDLE(b) // b -> c // c -> b // // package tree: // root // +-- a // | +-- b(1) // | +-- c(1) // +-- b(2) // +-- c(2) // 1. mark everything that's shadowed by anything in the bundle. This // marks b(2) and c(2). // 2. anything with edgesIn from outside the set, mark not-extraneous, // remove from set. This unmarks c(2). // 3. continue until no change // 4. remove everything in the set from the tree. b(2) is pruned // create the list of nodes shadowed by children of bundlers for (const bundles of bundlesByDepth.values()) { // skip the 'maxBundleDepth' item if (!Array.isArray(bundles)) { continue } for (const node of bundles) { for (const name of node.children.keys()) { const shadow = node.parent.resolve(name) if (!shadow) { continue } bundleShadowed.add(shadow) shadow.extraneous = true } } } // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3)) // a@1.2.3 -> (b@1.2.3) // a@1.3.0 -> (b@2) // b@1.2.3 -> () // b@2 -> (c@2) // // root // +-- lib // | +-- a@1.2.3 // | +-- b@1.2.3 // +-- b@2 <-- shadowed, now extraneous // +-- c@2 <-- also shadowed, because only dependent is shadowed for (const shadow of bundleShadowed) { for (const shadDep of shadow.edgesOut.values()) { /* istanbul ignore else - pretty unusual situation, just being * defensive here. Would mean that a bundled dep has a dependency * that is unmet. which, weird, but if you bundle it, we take * whatever you put there and assume the publisher knows best. */ if (shadDep.to) { bundleShadowed.add(shadDep.to) shadDep.to.extraneous = true } } } let changed do { changed = false for (const shadow of bundleShadowed) { for (const edge of shadow.edgesIn) { if (!bundleShadowed.has(edge.from)) { shadow.extraneous = false bundleShadowed.delete(shadow) changed = true break } } } } while (changed) for (const shadow of bundleShadowed) { this[_addNodeToTrashList](shadow) shadow.root = null } } async [_submitQuickAudit] () { if (this.options.audit === false) { this.auditReport = null return } // we submit the quick audit at this point in the process, as soon as // we have all the deps resolved, so that it can overlap with the other // actions as much as possible. Stash the promise, which we resolve // before finishing the reify() and returning the tree. Thus, we do // NOT return the promise, as the intent is for this to run in parallel // with the reification, and be resolved at a later time. const timeEnd = time.start('reify:audit') const options = { ...this.options } const tree = this.idealTree // if we're operating on a workspace, only audit the workspace deps if (this.options.workspaces.length) { options.filterSet = this.workspaceDependencySet( tree, this.options.workspaces, this.options.includeWorkspaceRoot ) } this.auditReport = AuditReport.load(tree, options).then(res => { timeEnd() return res }) } // ok! actually unpack stuff into their target locations! // The sparse tree has already been created, so we walk the diff // kicking off each unpack job. If any fail, we rm the sparse // tree entirely and try to put everything back where it was. [_unpackNewModules] () { const timeEnd = time.start('reify:unpack') const unpacks = [] dfwalk({ tree: this.diff, visit: diff => { // no unpacking if we don't want to change this thing if (diff.action !== 'CHANGE' && diff.action !== 'ADD') { return } const node = diff.ideal const bd = this.#bundleUnpacked.has(node) const sw = this.#shrinkwrapInflated.has(node) const bundleMissing = this.#bundleMissing.has(node) // check whether we still need to unpack this one. // test the inDepBundle last, since that's potentially a tree walk. const doUnpack = node && // can't unpack if removed! // root node already exists !node.isRoot && // already unpacked to read bundle !bd && // already unpacked to read sw !sw && // already unpacked by another dep's bundle (bundleMissing || !node.inDepBundle) if (doUnpack) { unpacks.push(this[_reifyNode](node)) } }, getChildren: diff => diff.children, }) return promiseAllRejectLate(unpacks).then(timeEnd) } // This is the part where we move back the unchanging nodes that were // the children of a node that did change. If this fails, the rollback // is a three-step process. First, we try to move the retired unchanged // nodes BACK to their retirement folders, then delete the sparse tree, // then move everything out of retirement. [_moveBackRetiredUnchanged] () { // get a list of all unchanging children of any shallow retired nodes // if they are not the ancestor of any node in the diff set, then the // directory won't already exist, so just rename it over. // This is sort of an inverse diff tree, of all the nodes where // the actualTree and idealTree _don't_ differ, starting from the // shallowest nodes that we moved aside in the first place. const timeEnd = time.start('reify:unretire') const moves = this.#retiredPaths this.#retiredUnchanged = {} return promiseAllRejectLate(this.diff.children.map(diff => { // skip if nothing was retired if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE') { return } const { path: realFolder } = diff.actual const retireFolder = moves[realFolder] /* istanbul ignore next - should be impossible */ debug(() => { if (!retireFolder) { const er = new Error('trying to un-retire but not retired') throw Object.assign(er, { realFolder, retireFolder, actual: diff.actual, ideal: diff.ideal, action: diff.action, }) } }) this.#retiredUnchanged[retireFolder] = [] return promiseAllRejectLate(diff.unchanged.map(node => { // no need to roll back links, since we'll just delete them anyway if (node.isLink) { return mkdir(dirname(node.path), { recursive: true, force: true }) .then(() => this[_reifyNode](node)) } // will have been moved/unpacked along with bundler if (node.inDepBundle && !this.#bundleMissing.has(node)) { return } this.#retiredUnchanged[retireFolder].push(node) const rel = relative(realFolder, node.path) const fromPath = resolve(retireFolder, rel) // if it has bundleDependencies, then make node_modules. otherwise // skip it. const bd = node.package.bundleDependencies const dir = bd && bd.length ? node.path + '/node_modules' : node.path return mkdir(dir, { recursive: true }).then(() => this[_moveContents](node, fromPath)) })) })).then(timeEnd) } // move the contents from the fromPath to the node.path [_moveContents] (node, fromPath) { return packageContents({ path: fromPath, depth: 1, packageJsonCache: new Map([[fromPath + '/package.json', node.package]]), }).then(res => promiseAllRejectLate(res.map(path => { const rel = relative(fromPath, path) const to = resolve(node.path, rel) return this[_renamePath](path, to) }))) } [_rollbackMoveBackRetiredUnchanged] (er) { const moves = this.#retiredPaths // flip the mapping around to go back const realFolders = new Map(Object.entries(moves).map(([k, v]) => [v, k])) const promises = Object.entries(this.#retiredUnchanged) .map(([retireFolder, nodes]) => promiseAllRejectLate(nodes.map(node => { const realFolder = realFolders.get(retireFolder) const rel = relative(realFolder, node.path) const fromPath = resolve(retireFolder, rel) return this[_moveContents]({ ...node, path: fromPath }, node.path) }))) return promiseAllRejectLate(promises) .then(() => this[_rollbackCreateSparseTree](er)) } [_build] () { const timeEnd = time.start('reify:build') // for all the things being installed, run their appropriate scripts // run in tip->root order, so as to be more likely to build a node's // deps before attempting to build it itself const nodes = [] dfwalk({ tree: this.diff, leave: diff => { if (!diff.ideal.isProjectRoot) { nodes.push(diff.ideal) } }, // process adds before changes, ignore removals getChildren: diff => diff && diff.children, filter: diff => diff.action === 'ADD' || diff.action === 'CHANGE', }) // pick up link nodes from the unchanged list as we want to run their // scripts in every install despite of having a diff status change for (const node of this.diff.unchanged) { const tree = node.root.target // skip links that only live within node_modules as they are most // likely managed by packages we installed, we only want to rebuild // unchanged links we directly manage const linkedFromRoot = node.parent === tree || node.target.fsTop === tree if (node.isLink && linkedFromRoot) { nodes.push(node) } } return this.rebuild({ nodes, handleOptionalFailure: true }).then(timeEnd) } // the tree is pretty much built now, so it's cleanup time. // remove the retired folders, and any deleted nodes // If this fails, there isn't much we can do but tell the user about it. // Thankfully, it's pretty unlikely that it'll fail, since rm is a node builtin. async [_removeTrash] () { const timeEnd = time.start('reify:trash') const promises = [] const failures = [] const _rm = path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er])) for (const path of this[_trashList]) { promises.push(_rm(path)) } await promiseAllRejectLate(promises) if (failures.length) { log.warn('cleanup', 'Failed to remove some directories', failures) } timeEnd() } // last but not least, we save the ideal tree metadata to the package-lock // or shrinkwrap file, and any additions or removals to package.json async [_saveIdealTree] (options) { // the ideal tree is actualized now, hooray! // it still contains all the references to optional nodes that were removed // for install failures. Those still end up in the shrinkwrap, so we // save it first, then prune out the optional trash, and then return it. const save = !(options.save === false) // we check for updates in order to make sure we run save ideal tree // even though save=false since we want `npm update` to be able to // write to package-lock files by default const hasUpdates = this[_updateAll] || this[_updateNames].length // we're going to completely skip save ideal tree in case of a global or // dry-run install and also if the save option is set to false, EXCEPT for // update since the expected behavior for npm7+ is for update to // NOT save to package.json, we make that exception since we still want // saveIdealTree to be able to write the lockfile by default. const saveIdealTree = !( (!save && !hasUpdates) || this.options.global || this.options.dryRun ) if (!saveIdealTree) { return false } const timeEnd = time.start('reify:save') const updatedTrees = new Set() const updateNodes = nodes => { for (const { name, tree: addTree } of nodes) { // addTree either the root, or a workspace const edge = addTree.edgesOut.get(name) const pkg = addTree.package const req = npa.resolve(name, edge.spec, addTree.realpath) const { rawSpec, subSpec } = req const spec = subSpec ? subSpec.rawSpec : rawSpec const child = edge.to // if we tried to install an optional dep, but it was a version // that we couldn't resolve, this MAY be missing. if we haven't // blown up by now, it's because it was not a problem, though, so // just move on. if (!child || !addTree.isTop) { continue } let newSpec // True if the dependency is getting installed from a local file path // In this case it is not possible to do the normal version comparisons // as the new version will be a file path const isLocalDep = req.type === 'directory' || req.type === 'file' if (req.registry) { const version = child.version const prefixRange = version ? this.options.savePrefix + version : '*' // if we installed a range, then we save the range specified // if it is not a subset of the ^x.y.z. eg, installing a range // of `1.x <1.2.3` will not be saved as `^1.2.0`, because that // would allow versions outside the requested range. Tags and // specific versions save with the save-prefix. const isRange = (subSpec || req).type === 'range' let range = spec if ( !isRange || spec === '*' || subset(prefixRange, spec, { loose: true }) ) { range = prefixRange } const pname = child.packageName const alias = name !== pname newSpec = alias ? `npm:${pname}@${range}` : range } else if (req.hosted) { // save the git+https url if it has auth, otherwise shortcut const h = req.hosted const opt = { noCommittish: false } if (h.https && h.auth) { newSpec = `git+${h.https(opt)}` } else { newSpec = h.shortcut(opt) } } else if (isLocalDep) { // when finding workspace nodes, make sure that // we save them using their version instead of // using their relative path if (edge.type === 'workspace') { const { version } = edge.to.target const prefixRange = version ? this.options.savePrefix + version : '*' newSpec = prefixRange } else { // save the relative path in package.json // Normally saveSpec is updated with the proper relative // path already, but it's possible to specify a full absolute // path initially, in which case we can end up with the wrong // thing, so just get the ultimate fetchSpec and relativize it. const p = req.fetchSpec.replace(/^file:/, '') const rel = relpath(addTree.realpath, p).replace(/#/g, '%23') newSpec = `file:${rel}` } } else { newSpec = req.saveSpec } if (options.saveType) { const depType = saveTypeMap.get(options.saveType) pkg[depType][name] = newSpec // rpj will have moved it here if it was in both // if it is empty it will be deleted later if (options.saveType === 'prod' && pkg.optionalDependencies) { delete pkg.optionalDependencies[name] } } else { if (hasSubKey(pkg, 'dependencies', name)) { pkg.dependencies[name] = newSpec } if (hasSubKey(pkg, 'devDependencies', name)) { pkg.devDependencies[name] = newSpec // don't update peer or optional if we don't have to if (hasSubKey(pkg, 'peerDependencies', name) && (isLocalDep || !intersects(newSpec, pkg.peerDependencies[name]))) { pkg.peerDependencies[name] = newSpec } if (hasSubKey(pkg, 'optionalDependencies', name) && (isLocalDep || !intersects(newSpec, pkg.optionalDependencies[name]))) { pkg.optionalDependencies[name] = newSpec } } else { if (hasSubKey(pkg, 'peerDependencies', name)) { pkg.peerDependencies[name] = newSpec } if (hasSubKey(pkg, 'optionalDependencies', name)) { pkg.optionalDependencies[name] = newSpec } } } updatedTrees.add(addTree) } } // Returns true if any of the edges from this node has a semver // range definition that is an exact match to the version installed // e.g: should return true if for a given an installed version 1.0.0, // range is either =1.0.0 or 1.0.0 const exactVersion = node => { for (const edge of node.edgesIn) { try { if (semver.subset(edge.spec, node.version)) { return false } } catch { // ignore errors } } return true } // helper that retrieves an array of nodes that were // potentially updated during the reify process, in order // to limit the number of nodes to check and update, only // select nodes from the inventory that are direct deps // of a given package.json (project root or a workspace) // and in ase of using a list of `names`, restrict nodes // to only names that are found in this list const retrieveUpdatedNodes = names => { const filterDirectDependencies = node => !node.isRoot && node.resolveParent && node.resolveParent.isRoot && (!names || names.includes(node.name)) && exactVersion(node) // skip update for exact ranges const directDeps = this.idealTree.inventory .filter(filterDirectDependencies) // traverses the list of direct dependencies and collect all nodes // to be updated, since any of them might have changed during reify const nodes = [] for (const node of directDeps) { for (const edgeIn of node.edgesIn) { nodes.push({ name: node.name, tree: edgeIn.from.target, }) } } return nodes } if (save) { // when using update all alongside with save, we'll make // sure to refresh every dependency of the root idealTree if (this[_updateAll]) { const nodes = retrieveUpdatedNodes() updateNodes(nodes) } else { // resolvedAdd is the list of user add requests, but with names added // to things like git repos and tarball file/urls. However, if the // user requested 'foo@', and we have a foo@file:../foo, then we should // end up saving the spec we actually used, not whatever they gave us. if (this[_resolvedAdd].length) { updateNodes(this[_resolvedAdd]) } // if updating given dependencies by name, restrict the list of // nodes to check to only those currently in _updateNames if (this[_updateNames].length) { const nodes = retrieveUpdatedNodes(this[_updateNames]) updateNodes(nodes) } // grab any from explicitRequests that had deps removed for (const { from: tree } of this.explicitRequests) { updatedTrees.add(tree) } } } if (save) { for (const tree of updatedTrees) { // refresh the edges so they have the correct specs tree.package = tree.package const pkgJson = await PackageJson.load(tree.path, { create: true }) const { dependencies = {}, devDependencies = {}, optionalDependencies = {}, peerDependencies = {}, // bundleDependencies is not required by PackageJson like the other // fields here PackageJson also doesn't omit an empty array for this // field so defaulting this to an empty array would add that field to // every package.json file. bundleDependencies, } = tree.package pkgJson.update({ dependencies, devDependencies, optionalDependencies, peerDependencies, bundleDependencies, }) await pkgJson.save() } } // before now edge specs could be changing, affecting the `requires` field // in the package lock, so we hold off saving to the very last action if (this[_usePackageLock]) { // preserve indentation, if possible let format = this.idealTree.package[Symbol.for('indent')] if (format === undefined) { format = ' ' } // TODO this ignores options.save await this.idealTree.meta.save({ format: (this.options.formatPackageLock && format) ? format : this.options.formatPackageLock, }) } timeEnd() return true } }