Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "rush-resolver-cache-plugin: add pnpm 10 / lockfile v9 compatibility",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
3 changes: 3 additions & 0 deletions common/config/subspaces/default/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions rush-plugins/rush-resolver-cache-plugin/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"_phase:test": "heft run --only test -- --clean"
},
"dependencies": {
"@pnpm/dependency-path": "1000.0.9",
"@rushstack/rush-sdk": "workspace:*"
},
"devDependencies": {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import { existsSync, readdirSync } from 'node:fs';

import type {
RushSession,
RushConfiguration,
Expand Down Expand Up @@ -79,7 +81,12 @@ export async function afterInstallAsync(

const lockFilePath: string = subspace.getCommittedShrinkwrapFilePath(variant);

const pnpmStoreDir: string = `${rushConfiguration.pnpmOptions.pnpmStorePath}/v3/files/`;
const pnpmStorePath: string = rushConfiguration.pnpmOptions.pnpmStorePath;
// pnpm 10 uses v10/index/ for index files; pnpm 8 uses v3/files/
const pnpmStoreV10IndexDir: string = `${pnpmStorePath}/v10/index/`;
const pnpmStoreV3FilesDir: string = `${pnpmStorePath}/v3/files/`;
const useV10Store: boolean = existsSync(pnpmStoreV10IndexDir);
const pnpmStoreDir: string = useV10Store ? pnpmStoreV10IndexDir : pnpmStoreV3FilesDir;

terminal.writeLine(`Using pnpm-lock from: ${lockFilePath}`);
terminal.writeLine(`Using pnpm store folder: ${pnpmStoreDir}`);
Expand Down Expand Up @@ -136,6 +143,42 @@ export async function afterInstallAsync(
// Ignore
}

/**
* Computes the pnpm store index file path for a given package integrity hash.
*/
function getStoreIndexPath(context: IResolverContext, hash: string): string {
if (!useV10Store) {
return `${pnpmStoreDir}${hash.slice(0, 2)}/${hash.slice(2)}-index.json`;
}

// pnpm 10 truncates integrity hashes to 32 bytes (64 hex chars) for index paths.
const truncHash: string = hash.length > 64 ? hash.slice(0, 64) : hash;
const hashDir: string = truncHash.slice(0, 2);
const hashRest: string = truncHash.slice(2);
// pnpm 10 index path format: <hash (0-2)>/<hash (2-64)>-<name>@<version>.json
const pkgName: string = (context.name || '').replace(/\//g, '+');
const nameVer: string = context.version ? `${pkgName}@${context.version}` : pkgName;
let indexPath: string = `${pnpmStoreDir}${hashDir}/${hashRest}-${nameVer}.json`;
// For truncated/hashed folder names, nameVer from the key may be wrong.
// Fallback: scan the directory for a file matching the hash prefix.
if (!existsSync(indexPath)) {
const dir: string = `${pnpmStoreDir}${hashDir}/`;
const filePrefix: string = `${hashRest}-`;
try {
const entries: import('node:fs').Dirent[] = readdirSync(dir, { withFileTypes: true });
const match: import('node:fs').Dirent | undefined = entries.find(
(e) => e.isFile() && e.name.startsWith(filePrefix)
);
if (match) {
indexPath = dir + match.name;
}
} catch {
// ignore
}
}
return indexPath;
}

async function afterExternalPackagesAsync(
contexts: Map<string, IResolverContext>,
missingOptionalDependencies: Set<string>
Expand Down Expand Up @@ -166,10 +209,7 @@ export async function afterInstallAsync(
const prefixIndex: number = descriptionFileHash.indexOf('-');
const hash: string = Buffer.from(descriptionFileHash.slice(prefixIndex + 1), 'base64').toString('hex');

// The pnpm store directory has index files of package contents at paths:
// <store>/v3/files/<hash (0-2)>/<hash (2-)>-index.json
// See https://github.com/pnpm/pnpm/blob/f394cfccda7bc519ceee8c33fc9b68a0f4235532/store/cafs/src/getFilePathInCafs.ts#L33
const indexPath: string = `${pnpmStoreDir}${hash.slice(0, 2)}/${hash.slice(2)}-index.json`;
const indexPath: string = getStoreIndexPath(context, hash);

try {
const indexContent: string = await FileSystem.readFileAsync(indexPath);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ import type {
} from '@rushstack/webpack-workspace-resolve-plugin';

import type { PnpmShrinkwrapFile } from './externals';
import { getDescriptionFileRootFromKey, resolveDependencies, createContextSerializer } from './helpers';
import {
getDescriptionFileRootFromKey,
resolveDependencies,
createContextSerializer,
extractNameAndVersionFromKey
} from './helpers';
import type { IResolverContext } from './types';

/**
Expand Down Expand Up @@ -182,9 +187,12 @@ export async function computeResolverCacheFromLockfileAsync(

const integrity: string | undefined = pack.resolution?.integrity;

if (!name && key.startsWith('/')) {
const versionIndex: number = key.indexOf('@', 2);
name = key.slice(1, versionIndex);
// Extract name and version from the key if not already provided
const parsed: { name: string; version: string } | undefined = extractNameAndVersionFromKey(key);
if (parsed) {
if (!name) {
name = parsed.name;
}
}

if (!name) {
Expand All @@ -196,6 +204,7 @@ export async function computeResolverCacheFromLockfileAsync(
descriptionFileHash: integrity,
isProject: false,
name,
version: parsed?.version,
deps: new Map(),
ordinal: -1,
optional: pack.optional
Expand All @@ -204,10 +213,10 @@ export async function computeResolverCacheFromLockfileAsync(
contexts.set(descriptionFileRoot, context);

if (pack.dependencies) {
resolveDependencies(workspaceRoot, pack.dependencies, context);
resolveDependencies(workspaceRoot, pack.dependencies, context, lockfile.packages);
}
if (pack.optionalDependencies) {
resolveDependencies(workspaceRoot, pack.optionalDependencies, context);
resolveDependencies(workspaceRoot, pack.optionalDependencies, context, lockfile.packages);
}
}

Expand Down Expand Up @@ -248,13 +257,13 @@ export async function computeResolverCacheFromLockfileAsync(
contexts.set(descriptionFileRoot, context);

if (importer.dependencies) {
resolveDependencies(workspaceRoot, importer.dependencies, context);
resolveDependencies(workspaceRoot, importer.dependencies, context, lockfile.packages);
}
if (importer.devDependencies) {
resolveDependencies(workspaceRoot, importer.devDependencies, context);
resolveDependencies(workspaceRoot, importer.devDependencies, context, lockfile.packages);
}
if (importer.optionalDependencies) {
resolveDependencies(workspaceRoot, importer.optionalDependencies, context);
resolveDependencies(workspaceRoot, importer.optionalDependencies, context, lockfile.packages);
}
}

Expand Down
101 changes: 35 additions & 66 deletions rush-plugins/rush-resolver-cache-plugin/src/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,58 +1,15 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import { createHash } from 'node:crypto';
import * as path from 'node:path';

import { depPathToFilename } from '@pnpm/dependency-path';

import type { ISerializedResolveContext } from '@rushstack/webpack-workspace-resolve-plugin';

import type { IDependencyEntry, IResolverContext } from './types';

const MAX_LENGTH_WITHOUT_HASH: number = 120 - 26 - 1;
const BASE32: string[] = 'abcdefghijklmnopqrstuvwxyz234567'.split('');

// https://github.com/swansontec/rfc4648.js/blob/ead9c9b4b68e5d4a529f32925da02c02984e772c/src/codec.ts#L82-L118
export function createBase32Hash(input: string): string {
const data: Buffer = createHash('md5').update(input).digest();

const mask: 0x1f = 0x1f;
let out: string = '';

let bits: number = 0; // Number of bits currently in the buffer
let buffer: number = 0; // Bits waiting to be written out, MSB first
for (let i: number = 0; i < data.length; ++i) {
// eslint-disable-next-line no-bitwise
buffer = (buffer << 8) | (0xff & data[i]);
bits += 8;

// Write out as much as we can:
while (bits > 5) {
bits -= 5;
// eslint-disable-next-line no-bitwise
out += BASE32[mask & (buffer >> bits)];
}
}

// Partial character:
if (bits) {
// eslint-disable-next-line no-bitwise
out += BASE32[mask & (buffer << (5 - bits))];
}

return out;
}

// https://github.com/pnpm/pnpm/blob/f394cfccda7bc519ceee8c33fc9b68a0f4235532/packages/dependency-path/src/index.ts#L167-L189
export function depPathToFilename(depPath: string): string {
let filename: string = depPathToFilenameUnescaped(depPath).replace(/[\\/:*?"<>|]/g, '+');
if (filename.includes('(')) {
filename = filename.replace(/(\)\()|\(/g, '_').replace(/\)$/, '');
}
if (filename.length > 120 || (filename !== filename.toLowerCase() && !filename.startsWith('file+'))) {
return `${filename.substring(0, MAX_LENGTH_WITHOUT_HASH)}_${createBase32Hash(filename)}`;
}
return filename;
}
const PNPM_STORE_DIR_MAX_LENGTH: number = 120;

/**
* Computes the root folder for a dependency from a reference to it in another package
Expand All @@ -66,20 +23,24 @@ export function resolveDependencyKey(
lockfileFolder: string,
key: string,
specifier: string,
context: IResolverContext
context: IResolverContext,
packageKeys?: { has(key: string): boolean }
): string {
if (specifier.startsWith('/')) {
return getDescriptionFileRootFromKey(lockfileFolder, specifier);
} else if (specifier.startsWith('link:')) {
if (specifier.startsWith('link:')) {
if (context.isProject) {
return path.posix.join(context.descriptionFileRoot, specifier.slice(5));
} else {
return path.posix.join(lockfileFolder, specifier.slice(5));
}
} else if (specifier.startsWith('file:')) {
return getDescriptionFileRootFromKey(lockfileFolder, specifier, key);
} else if (packageKeys?.has(specifier)) {
// The specifier is a full package key
return getDescriptionFileRootFromKey(lockfileFolder, specifier);
} else {
return getDescriptionFileRootFromKey(lockfileFolder, `/${key}@${specifier}`);
// Construct the full dependency key from package name and version specifier.
const fullKey: string = `${key}@${specifier}`;
return getDescriptionFileRootFromKey(lockfileFolder, fullKey);
}
}

Expand All @@ -91,44 +52,52 @@ export function resolveDependencyKey(
* @returns The physical path to the dependency
*/
export function getDescriptionFileRootFromKey(lockfileFolder: string, key: string, name?: string): string {
if (!key.startsWith('file:')) {
name = key.slice(1, key.indexOf('@', 2));
if (!key.startsWith('file:') && !name) {
const offset: number = key.startsWith('/') ? 1 : 0;
name = key.slice(offset, key.indexOf('@', offset + 1));
}
if (!name) {
throw new Error(`Missing package name for ${key}`);
}

const originFolder: string = `${lockfileFolder}/node_modules/.pnpm/${depPathToFilename(key)}/node_modules`;
const originFolder: string = `${lockfileFolder}/node_modules/.pnpm/${depPathToFilename(key, PNPM_STORE_DIR_MAX_LENGTH)}/node_modules`;
const descriptionFileRoot: string = `${originFolder}/${name}`;
return descriptionFileRoot;
}

export function resolveDependencies(
lockfileFolder: string,
collection: Record<string, IDependencyEntry>,
context: IResolverContext
context: IResolverContext,
packageKeys?: { has(key: string): boolean }
): void {
for (const [key, value] of Object.entries(collection)) {
const version: string = typeof value === 'string' ? value : value.version;
const resolved: string = resolveDependencyKey(lockfileFolder, key, version, context);
const resolved: string = resolveDependencyKey(lockfileFolder, key, version, context, packageKeys);

context.deps.set(key, resolved);
}
}

/**
*
* @param depPath - The path to the dependency
* @returns The folder name for the dependency
* Extracts the package name and version from a lockfile package key.
* @param key - The lockfile package key (e.g. '/autoprefixer\@9.8.8', '\@scope/name\@1.0.0(peer\@2.0.0)')
* @returns The extracted name and version, or undefined for file: keys
*/
export function depPathToFilenameUnescaped(depPath: string): string {
if (depPath.indexOf('file:') !== 0) {
if (depPath.startsWith('/')) {
depPath = depPath.slice(1);
}
return depPath;
export function extractNameAndVersionFromKey(key: string): { name: string; version: string } | undefined {
if (key.startsWith('file:')) {
return undefined;
}
const offset: number = key.startsWith('/') ? 1 : 0;
const versionAtIndex: number = key.indexOf('@', offset + 1);
if (versionAtIndex === -1) {
return undefined;
}
return depPath.replace(':', '+');
const name: string = key.slice(offset, versionAtIndex);
const parenIndex: number = key.indexOf('(', versionAtIndex);
const version: string =
parenIndex !== -1 ? key.slice(versionAtIndex + 1, parenIndex) : key.slice(versionAtIndex + 1);
return { name, version };
}

/**
Expand Down
Loading
Loading