feat: configurable max length of directories inside node_modules/.pnpm (

pnpm#7994)

close pnpm#7355

.changeset/old-cars-notice.md

@@ -0,0 +1,37 @@
+---
+"@pnpm/dependency-path": major
+"@pnpm/plugin-commands-installation": minor
+"@pnpm/plugin-commands-publishing": minor
+"@pnpm/plugin-commands-script-runners": minor
+"@pnpm/plugin-commands-licenses": minor
+"@pnpm/plugin-commands-outdated": minor
+"@pnpm/plugin-commands-patching": minor
+"@pnpm/read-projects-context": minor
+"@pnpm/plugin-commands-listing": minor
+"@pnpm/resolve-dependencies": minor
+"@pnpm/plugin-commands-deploy": minor
+"@pnpm/reviewing.dependencies-hierarchy": minor
+"@pnpm/plugin-commands-audit": minor
+"@pnpm/store-connection-manager": minor
+"@pnpm/package-requester": minor
+"@pnpm/plugin-commands-rebuild": minor
+"@pnpm/modules-cleaner": minor
+"@pnpm/plugin-commands-store": minor
+"@pnpm/license-scanner": minor
+"@pnpm/lockfile-to-pnp": minor
+"@pnpm/modules-yaml": minor
+"@pnpm/lockfile-utils": minor
+"@pnpm/get-context": minor
+"@pnpm/mount-modules": minor
+"@pnpm/headless": minor
+"@pnpm/package-store": minor
+"@pnpm/deps.graph-builder": minor
+"@pnpm/hoist": minor
+"@pnpm/core": minor
+"@pnpm/audit": minor
+"@pnpm/list": minor
+"@pnpm/config": minor
+"@pnpm/server": minor
+---
+
+New setting called `virtual-store-dir-max-length` added for modifying the max allowed length of the directories inside `node_modules/.pnpm`. The default length is 120 characters [#7355](https://github.com/pnpm/pnpm/issues/7355).

config/config/src/Config.ts

@@ -192,6 +192,7 @@ export interface Config {
   dedupeInjectedDeps?: boolean
   nodeOptions?: string
   packageManagerStrict?: boolean
+  virtualStoreDirMaxLength: number
 }
 
 export interface ConfigWithDeprecatedSettings extends Config {

config/config/src/index.ts

@@ -140,6 +140,7 @@ export const types = Object.assign({
   'use-stderr': Boolean,
   'verify-store-integrity': Boolean,
   'virtual-store-dir': String,
+  'virtual-store-dir-max-length': Number,
   'workspace-concurrency': Number,
   'workspace-packages': [String, Array],
   'workspace-root': Boolean,
@@ -272,6 +273,7 @@ export async function getConfig (
     'workspace-prefix': opts.workspaceDir,
     'embed-readme': false,
     'registry-supports-time-field': false,
+    'virtual-store-dir-max-length': 120,
   }
 
   const { config: npmConfig, warnings, failedToLoadBuiltInConfig } = loadNpmConf(cliOptions, rcOptionsTypes, defaultOptions)

deps/graph-builder/src/lockfileToDepGraph.ts

@@ -68,6 +68,7 @@ export interface LockfileToDepGraphOptions {
   storeDir: string
   virtualStoreDir: string
   supportedArchitectures?: SupportedArchitectures
+  virtualStoreDirMaxLength: number
 }
 
 export interface DirectDependenciesByImporterId {
@@ -103,7 +104,7 @@ export async function lockfileToDepGraph (
         if (opts.skipped.has(depPath)) return
         // TODO: optimize. This info can be already returned by pkgSnapshotToResolution()
         const { name: pkgName, version: pkgVersion } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
-        const modules = path.join(opts.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
+        const modules = path.join(opts.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules')
         const packageId = packageIdFromSnapshot(depPath, pkgSnapshot)
 
         const pkg = {
@@ -208,6 +209,7 @@ export async function lockfileToDepGraph (
       storeController: opts.storeController,
       storeDir: opts.storeDir,
       virtualStoreDir: opts.virtualStoreDir,
+      virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
     }
     for (const [dir, node] of Object.entries(graph)) {
       const pkgSnapshot = pkgSnapshotByLocation[dir]
@@ -244,6 +246,7 @@ function getChildrenPaths (
     lockfileDir: string
     sideEffectsCacheRead: boolean
     storeController: StoreController
+    virtualStoreDirMaxLength: number
   },
   allDeps: { [alias: string]: string },
   peerDeps: Set<string> | null,
@@ -263,7 +266,7 @@ function getChildrenPaths (
     } else if (childPkgSnapshot) {
       if (ctx.skipped.has(childRelDepPath)) continue
       const pkgName = nameVerFromPkgSnapshot(childRelDepPath, childPkgSnapshot).name
-      children[alias] = path.join(ctx.virtualStoreDir, dp.depPathToFilename(childRelDepPath), 'node_modules', pkgName)
+      children[alias] = path.join(ctx.virtualStoreDir, dp.depPathToFilename(childRelDepPath, ctx.virtualStoreDirMaxLength), 'node_modules', pkgName)
     } else if (ref.indexOf('file:') === 0) {
       children[alias] = path.resolve(ctx.lockfileDir, ref.slice(5))
     } else if (!ctx.skipped.has(childRelDepPath) && ((peerDeps == null) || !peerDeps.has(alias))) {

exec/plugin-commands-rebuild/src/implementation/extendRebuildOptions.ts

@@ -47,6 +47,7 @@ export type StrictRebuildOptions = {
   deployAllFiles: boolean
   neverBuiltDependencies?: string[]
   onlyBuiltDependencies?: string[]
+  virtualStoreDirMaxLength: number
 } & Pick<Config, 'sslConfigs'>
 
 export type RebuildOptions = Partial<StrictRebuildOptions> &

exec/plugin-commands-rebuild/src/implementation/index.ts

@@ -200,6 +200,7 @@ export async function rebuildProjects (
     skipped: Array.from(ctx.skipped),
     storeDir: ctx.storeDir,
     virtualStoreDir: ctx.virtualStoreDir,
+    virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
   })
 }
 
@@ -295,7 +296,7 @@ async function _rebuild (
       const pkgInfo = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
       const pkgRoots = opts.nodeLinker === 'hoisted'
         ? (ctx.modulesFile?.hoistedLocations?.[depPath] ?? []).map((hoistedLocation) => path.join(opts.lockfileDir, hoistedLocation))
-        : [path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules', pkgInfo.name)]
+        : [path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules', pkgInfo.name)]
       if (pkgRoots.length === 0) {
         if (pkgSnapshot.optional) return
         throw new PnpmError('MISSING_HOISTED_LOCATIONS', `${depPath} is not found in hoistedLocations inside node_modules/.modules.yaml`, {
@@ -306,7 +307,7 @@ async function _rebuild (
       try {
         const extraBinPaths = ctx.extraBinPaths
         if (opts.nodeLinker !== 'hoisted') {
-          const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
+          const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules')
           const binPath = path.join(pkgRoot, 'node_modules', '.bin')
           await linkBins(modules, binPath, { extraNodePaths: ctx.extraNodePaths, warn })
         } else {
@@ -403,7 +404,7 @@ async function _rebuild (
         .map(async (depPath) => limitLinking(async () => {
           const pkgSnapshot = pkgSnapshots[depPath]
           const pkgInfo = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
-          const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
+          const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules')
           const binPath = path.join(modules, pkgInfo.name, 'node_modules', '.bin')
           return linkBins(modules, binPath, { warn })
         }))

exec/plugin-commands-rebuild/test/utils/index.ts

@@ -52,4 +52,5 @@ export const DEFAULT_OPTS = {
     cpu: ['current'],
     libc: ['current'],
   },
+  virtualStoreDirMaxLength: 120,
 }

exec/plugin-commands-script-runners/test/utils/index.ts

@@ -55,6 +55,7 @@ export const DEFAULT_OPTS = {
     cpu: ['current'],
     libc: ['current'],
   },
+  virtualStoreDirMaxLength: 120,
 }
 
 export const DLX_DEFAULT_OPTS = {
@@ -91,4 +92,5 @@ export const DLX_DEFAULT_OPTS = {
     cpu: ['current'],
     libc: ['current'],
   },
+  virtualStoreDirMaxLength: 120,
 }

lockfile/audit/src/index.ts

@@ -23,6 +23,7 @@ export async function audit (
     registry: string
     retry?: RetryTimeoutOptions
     timeout?: number
+    virtualStoreDirMaxLength: number
   }
 ): Promise<AuditReport> {
   const auditTree = await lockfileToAuditTree(lockfile, { include: opts.include, lockfileDir: opts.lockfileDir })
@@ -55,6 +56,7 @@ export async function audit (
       lockfile,
       lockfileDir: opts.lockfileDir,
       include: opts.include,
+      virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
     })
   } catch (err: unknown) {
     assert(util.types.isNativeError(err))
@@ -79,6 +81,7 @@ async function extendWithDependencyPaths (auditReport: AuditReport, opts: {
   lockfile: Lockfile
   lockfileDir: string
   include?: { [dependenciesField in DependenciesField]: boolean }
+  virtualStoreDirMaxLength: number
 }): Promise<AuditReport> {
   const { advisories } = auditReport
   if (!Object.keys(advisories).length) return auditReport
@@ -88,6 +91,7 @@ async function extendWithDependencyPaths (auditReport: AuditReport, opts: {
     lockfileDir: opts.lockfileDir,
     depth: Infinity,
     include: opts.include,
+    virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
   }
   const _searchPackagePaths = searchPackagePaths.bind(null, searchOpts, projectDirs)
   // eslint-disable-next-line @typescript-eslint/naming-convention
@@ -104,6 +108,7 @@ async function searchPackagePaths (
     lockfileDir: string
     depth: number
     include?: { [dependenciesField in DependenciesField]: boolean }
+    virtualStoreDirMaxLength: number
   },
   projectDirs: string[],
   pkg: string

lockfile/audit/test/index.ts

@@ -164,6 +164,7 @@ describe('audit', () => {
         retry: {
           retries: 0,
         },
+        virtualStoreDirMaxLength: 120,
       })
     } catch (_err: any) { // eslint-disable-line
       err = _err

lockfile/lockfile-to-pnp/src/index.ts

@@ -15,6 +15,7 @@ export async function writePnpFile (
     importerNames: Record<string, string>
     lockfileDir: string
     virtualStoreDir: string
+    virtualStoreDirMaxLength: number
     registries: Registries
   }
 ): Promise<void> {
@@ -36,6 +37,7 @@ export function lockfileToPackageRegistry (
     importerNames: { [importerId: string]: string }
     lockfileDir: string
     virtualStoreDir: string
+    virtualStoreDirMaxLength: number
     registries: Registries
   }
 ): PackageRegistry {
@@ -87,7 +89,7 @@ export function lockfileToPackageRegistry (
     // Seems like this field should always contain a relative path
     let packageLocation = normalizePath(path.relative(opts.lockfileDir, path.join(
       opts.virtualStoreDir,
-      depPathToFilename(relDepPath),
+      depPathToFilename(relDepPath, opts.virtualStoreDirMaxLength),
       'node_modules',
       name
     )))

lockfile/lockfile-to-pnp/test/index.ts

@@ -61,6 +61,7 @@ test('lockfileToPackageRegistry', () => {
       default: 'https://registry.npmjs.org/',
     },
     virtualStoreDir: path.resolve('node_modules/.pnpm'),
+    virtualStoreDirMaxLength: 120,
   })
 
   const actual = Array.from(
@@ -212,6 +213,7 @@ test('lockfileToPackageRegistry packages that have peer deps', () => {
       default: 'https://registry.npmjs.org/',
     },
     virtualStoreDir: path.resolve('node_modules/.pnpm'),
+    virtualStoreDirMaxLength: 120,
   })
 
   const actual = Array.from(

lockfile/lockfile-utils/src/extendProjectsWithTargetDirs.ts

@@ -12,11 +12,12 @@ export function extendProjectsWithTargetDirs<T> (
   ctx: {
     virtualStoreDir: string
     pkgLocationsByDepPath?: Record<string, string[]>
+    virtualStoreDirMaxLength: number
   }
 ): Array<T & { id: string, stages: string[], targetDirs: string[] }> {
   const getLocalLocations: GetLocalLocations = ctx.pkgLocationsByDepPath != null
     ? (depPath: string) => ctx.pkgLocationsByDepPath![depPath]
-    : (depPath: string, pkgName: string) => [path.join(ctx.virtualStoreDir, depPathToFilename(depPath), 'node_modules', pkgName)]
+    : (depPath: string, pkgName: string) => [path.join(ctx.virtualStoreDir, depPathToFilename(depPath, ctx.virtualStoreDirMaxLength), 'node_modules', pkgName)]
   const projectsById: Record<string, T & { id: string, targetDirs: string[], stages?: string[] }> =
     Object.fromEntries(projects.map((project) => [project.id, { ...project, targetDirs: [] as string[] }]))
   Object.entries(lockfile.packages ?? {})

lockfile/plugin-commands-audit/src/audit.ts

@@ -141,6 +141,7 @@ export async function handler (
   | 'userConfig'
   | 'rawConfig'
   | 'rootProjectManifest'
+  | 'virtualStoreDirMaxLength'
   >
 ): Promise<{ exitCode: number, output: string }> {
   const lockfileDir = opts.lockfileDir ?? opts.dir
@@ -179,6 +180,7 @@ export async function handler (
         retries: opts.fetchRetries,
       },
       timeout: opts.fetchTimeout,
+      virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
     })
   } catch (err: any) { // eslint-disable-line
     if (opts.ignoreRegistryErrors) {

lockfile/plugin-commands-audit/test/fix.ts

@@ -29,6 +29,7 @@ test('overrides are added for vulnerable dependencies', async () => {
     userConfig: {},
     rawConfig,
     registries,
+    virtualStoreDirMaxLength: 120,
   })
 
   expect(exitCode).toBe(0)
@@ -53,6 +54,7 @@ test('no overrides are added if no vulnerabilities are found', async () => {
     userConfig: {},
     rawConfig,
     registries,
+    virtualStoreDirMaxLength: 120,
   })
 
   expect(exitCode).toBe(0)
@@ -88,6 +90,7 @@ test('CVEs found in the allow list are not added as overrides', async () => {
     userConfig: {},
     rawConfig,
     registries,
+    virtualStoreDirMaxLength: 120,
   })
   expect(exitCode).toBe(0)
   expect(output).toMatch(/Run "pnpm install"/)