From 9d71b9706260000332541c0c1565177c274f05e5 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 12 Aug 2024 19:12:31 +0000 Subject: [PATCH 1/5] [workspace-resolve-plugin] Create plugin --- ...space-resolve-plugin_2024-08-12-19-12.json | 10 + .../config/subspaces/default/pnpm-lock.yaml | 25 ++ .../webpack-workspace-resolve-plugin.api.md | 58 +++++ rush.json | 6 + .../.eslintrc.js | 12 + .../.npmignore | 32 +++ .../webpack-workspace-resolve-plugin/LICENSE | 24 ++ .../README.md | 35 +++ .../config/api-extractor.json | 19 ++ .../config/jest.config.json | 3 + .../config/rig.json | 7 + .../package.json | 42 ++++ .../src/KnownDescriptionFilePlugin.ts | 121 ++++++++++ .../src/KnownPackageDependenciesPlugin.ts | 72 ++++++ .../src/WorkspaceLayoutCache.ts | 166 ++++++++++++++ .../src/WorkspaceResolvePlugin.ts | 65 ++++++ .../src/index.ts | 11 + .../src/normalizeSlashes.ts | 10 + .../test/KnownDescriptionFilePlugin.test.ts | 167 ++++++++++++++ .../KnownPackageDependenciesPlugin.test.ts | 214 ++++++++++++++++++ .../tsconfig.json | 7 + 21 files changed, 1106 insertions(+) create mode 100644 common/changes/@rushstack/webpack-workspace-resolve-plugin/workspace-resolve-plugin_2024-08-12-19-12.json create mode 100644 common/reviews/api/webpack-workspace-resolve-plugin.api.md create mode 100644 webpack/webpack-workspace-resolve-plugin/.eslintrc.js create mode 100644 webpack/webpack-workspace-resolve-plugin/.npmignore create mode 100644 webpack/webpack-workspace-resolve-plugin/LICENSE create mode 100644 webpack/webpack-workspace-resolve-plugin/README.md create mode 100644 webpack/webpack-workspace-resolve-plugin/config/api-extractor.json create mode 100644 webpack/webpack-workspace-resolve-plugin/config/jest.config.json create mode 100644 webpack/webpack-workspace-resolve-plugin/config/rig.json create mode 100644 webpack/webpack-workspace-resolve-plugin/package.json create mode 100644 webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/index.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/tsconfig.json diff --git a/common/changes/@rushstack/webpack-workspace-resolve-plugin/workspace-resolve-plugin_2024-08-12-19-12.json b/common/changes/@rushstack/webpack-workspace-resolve-plugin/workspace-resolve-plugin_2024-08-12-19-12.json new file mode 100644 index 00000000000..3fe1f92ec33 --- /dev/null +++ b/common/changes/@rushstack/webpack-workspace-resolve-plugin/workspace-resolve-plugin_2024-08-12-19-12.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/webpack-workspace-resolve-plugin", + "comment": "Add plugin for more efficient import resolution in a monorepo with known structure. Optimizes lookup of the relevant `package.json` for a given path, and lookup of npm dependencies of the containing package.", + "type": "minor" + } + ], + "packageName": "@rushstack/webpack-workspace-resolve-plugin" +} \ No newline at end of file diff --git a/common/config/subspaces/default/pnpm-lock.yaml b/common/config/subspaces/default/pnpm-lock.yaml index 5423702e50a..48d0cb9879f 100644 --- a/common/config/subspaces/default/pnpm-lock.yaml +++ b/common/config/subspaces/default/pnpm-lock.yaml @@ -4401,6 +4401,31 @@ importers: specifier: ~5.82.1 version: 5.82.1 + ../../../webpack/webpack-workspace-resolve-plugin: + dependencies: + '@rushstack/lookup-by-path': + specifier: workspace:* + version: link:../../libraries/lookup-by-path + '@types/tapable': + specifier: 1.0.6 + version: 1.0.6 + tapable: + specifier: 2.2.1 + version: 2.2.1 + devDependencies: + '@rushstack/heft': + specifier: workspace:* + version: link:../../apps/heft + local-node-rig: + specifier: workspace:* + version: link:../../rigs/local-node-rig + memfs: + specifier: 3.4.3 + version: 3.4.3 + webpack: + specifier: ~5.82.1 + version: 5.82.1 + ../../../webpack/webpack4-localization-plugin: dependencies: '@rushstack/localization-utilities': diff --git a/common/reviews/api/webpack-workspace-resolve-plugin.api.md b/common/reviews/api/webpack-workspace-resolve-plugin.api.md new file mode 100644 index 00000000000..c47850e2591 --- /dev/null +++ b/common/reviews/api/webpack-workspace-resolve-plugin.api.md @@ -0,0 +1,58 @@ +## API Report File for "@rushstack/webpack-workspace-resolve-plugin" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import type { Compiler } from 'webpack'; +import { IPrefixMatch } from '@rushstack/lookup-by-path'; +import { LookupByPath } from '@rushstack/lookup-by-path'; +import type { WebpackPluginInstance } from 'webpack'; + +// @beta +export interface IResolveContext { + descriptionFileRoot: string; + findDependency(request: string): IPrefixMatch | undefined; +} + +// @beta +export interface IResolverCacheFile { + contexts: ISerializedResolveContext[]; +} + +// @beta +export interface ISerializedResolveContext { + deps: Record; + dirInfoFiles?: string[]; + name: string; + root: string; +} + +// @beta +export interface IWorkspaceLayoutCacheOptions { + cacheData: IResolverCacheFile; + workspaceRoot: string; +} + +// @beta +export interface IWorkspaceResolvePluginOptions { + cache: WorkspaceLayoutCache; +} + +// @beta +export class WorkspaceLayoutCache { + constructor(options: IWorkspaceLayoutCacheOptions); + readonly contextForPackage: WeakMap; + readonly contextLookup: LookupByPath; +} + +// @beta +export class WorkspaceResolvePlugin implements WebpackPluginInstance { + constructor(cache: WorkspaceLayoutCache); + // (undocumented) + apply(compiler: Compiler): void; +} + +// (No @packageDocumentation comment for this package) + +``` diff --git a/rush.json b/rush.json index 5728d6e9eb8..d6a547cecf3 100644 --- a/rush.json +++ b/rush.json @@ -1275,6 +1275,12 @@ "reviewCategory": "libraries", "shouldPublish": false // for now }, + { + "packageName": "@rushstack/webpack-workspace-resolve-plugin", + "projectFolder": "webpack/webpack-workspace-resolve-plugin", + "reviewCategory": "libraries", + "shouldPublish": true + }, { "packageName": "@rushstack/set-webpack-public-path-plugin", "projectFolder": "webpack/set-webpack-public-path-plugin", diff --git a/webpack/webpack-workspace-resolve-plugin/.eslintrc.js b/webpack/webpack-workspace-resolve-plugin/.eslintrc.js new file mode 100644 index 00000000000..27dc0bdff95 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/.eslintrc.js @@ -0,0 +1,12 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('local-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution'); +// This is a workaround for https://github.com/microsoft/rushstack/issues/3021 +require('local-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names'); + +module.exports = { + extends: [ + 'local-node-rig/profiles/default/includes/eslint/profile/node-trusted-tool', + 'local-node-rig/profiles/default/includes/eslint/mixins/friendly-locals' + ], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/webpack/webpack-workspace-resolve-plugin/.npmignore b/webpack/webpack-workspace-resolve-plugin/.npmignore new file mode 100644 index 00000000000..bc349f9a4be --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/.npmignore @@ -0,0 +1,32 @@ +# THIS IS A STANDARD TEMPLATE FOR .npmignore FILES IN THIS REPO. + +# Ignore all files by default, to avoid accidentally publishing unintended files. +* + +# Use negative patterns to bring back the specific things we want to publish. +!/bin/** +!/lib/** +!/lib-*/** +!/dist/** + +!CHANGELOG.md +!CHANGELOG.json +!heft-plugin.json +!rush-plugin-manifest.json +!ThirdPartyNotice.txt + +# Ignore certain patterns that should not get published. +/dist/*.stats.* +/lib/**/test/ +/lib-*/**/test/ +*.test.js + +# NOTE: These don't need to be specified, because NPM includes them automatically. +# +# package.json +# README.md +# LICENSE + +# --------------------------------------------------------------------------- +# DO NOT MODIFY ABOVE THIS LINE! Add any project-specific overrides below. +# --------------------------------------------------------------------------- diff --git a/webpack/webpack-workspace-resolve-plugin/LICENSE b/webpack/webpack-workspace-resolve-plugin/LICENSE new file mode 100644 index 00000000000..42b9592b121 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/LICENSE @@ -0,0 +1,24 @@ +@rushstack/webpack-workspace-resolve-plugin + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/webpack/webpack-workspace-resolve-plugin/README.md b/webpack/webpack-workspace-resolve-plugin/README.md new file mode 100644 index 00000000000..7397a279f45 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/README.md @@ -0,0 +1,35 @@ +# @rushstack/webpack-workspace-resolve-plugin + +This package contains a plugin for webpack 5 that leverages a cache file generated from package manager metadata to greatly accelerate module resolution. +Local benchmarks have shown a savings of around 10% of build time for some fairly large closed source projects. + +## Installation + +`npm install @rushstack/webpack-workspace-resolve-plugin --save-dev` + +## Overview + +This plugin is intended primarily for use in pnpm monorepos, but any tool that produces a strict package layout can be made compatible by generating the necessary cache file. + +The cache file contains information about the locations of every `package.json` file known to the package manager (including those in subdirectories of packages), as well as the list of declared dependencies of each and where they can be found. + +When using this plugin, the following options should be configured for your resolver: +- `symlinks: false` - Since the cache knows the symlinks for package dependencies, you can avoid the cost of testing for other symlinks unless you are using additional symlinks. +- `modules: []` - The cache should contain all information necessary to locate available dependencies for any arbitrary folder. If you need to allow resolution in other roots, you can add those, but omit `'node_modules'`. + +## Limitations + +This plugin depends on the presence of a cache file in the workspace to function. Data in this cache file is assumed not to change while the webpack process is running, although the file will be. + +**Note:** Generating the cache file is not in the scope of this plugin. + +This plugin does not currently support having subdirectory `package.json` files within workspace projects (e.g. for declaring `{ "type": "module" }` in mixed CommonJS/ESM packages). +This plugin does not work (well) with a hoisted node_modules installation layout. + +## Links + +- [CHANGELOG.md]( + https://github.com/microsoft/rushstack/blob/main/webpack/webpack-workspace-resolve-plugin/CHANGELOG.md) - Find + out what's new in the latest version + +`@rushstack/webpack5-workspace-resolve-plugin` is part of the [Rush Stack](https://rushstack.io/) family of projects. diff --git a/webpack/webpack-workspace-resolve-plugin/config/api-extractor.json b/webpack/webpack-workspace-resolve-plugin/config/api-extractor.json new file mode 100644 index 00000000000..fba8a2992f6 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/config/api-extractor.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + "mainEntryPointFilePath": "/lib/index.d.ts", + + "apiReport": { + "enabled": true, + "reportFolder": "../../../common/reviews/api" + }, + + "docModel": { + "enabled": false, + "apiJsonFilePath": "../../../common/temp/api/.api.json" + }, + + "dtsRollup": { + "enabled": true + } +} diff --git a/webpack/webpack-workspace-resolve-plugin/config/jest.config.json b/webpack/webpack-workspace-resolve-plugin/config/jest.config.json new file mode 100644 index 00000000000..d1749681d90 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/config/jest.config.json @@ -0,0 +1,3 @@ +{ + "extends": "local-node-rig/profiles/default/config/jest.config.json" +} diff --git a/webpack/webpack-workspace-resolve-plugin/config/rig.json b/webpack/webpack-workspace-resolve-plugin/config/rig.json new file mode 100644 index 00000000000..165ffb001f5 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/config/rig.json @@ -0,0 +1,7 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + + "rigPackageName": "local-node-rig" +} diff --git a/webpack/webpack-workspace-resolve-plugin/package.json b/webpack/webpack-workspace-resolve-plugin/package.json new file mode 100644 index 00000000000..8c521454c80 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/package.json @@ -0,0 +1,42 @@ +{ + "name": "@rushstack/webpack-workspace-resolve-plugin", + "version": "0.0.0", + "description": "This plugin leverages workspace-level metadata to greatly accelerate module resolution.", + "main": "lib/index.js", + "typings": "dist/webpack-workspace-resolve-plugin.d.ts", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/rushstack.git", + "directory": "webpack/webpack-workspace-resolve-plugin" + }, + "engines": { + "node": ">=18.19.0" + }, + "scripts": { + "build": "heft build --clean", + "_phase:build": "heft run --only build -- --clean", + "_phase:test": "heft run --only test -- --clean" + }, + "peerDependencies": { + "webpack": "^5.68.0", + "@types/node": "*" + }, + "dependencies": { + "@rushstack/lookup-by-path": "workspace:*", + "@types/tapable": "1.0.6", + "tapable": "2.2.1" + }, + "devDependencies": { + "@rushstack/heft": "workspace:*", + "local-node-rig": "workspace:*", + "memfs": "3.4.3", + "webpack": "~5.82.1" + }, + "sideEffects": false, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } +} diff --git a/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts new file mode 100644 index 00000000000..6bbdc299857 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { sep as directorySeparator } from 'node:path'; + +import type { Resolver } from 'webpack'; +import type { IPrefixMatch } from '@rushstack/lookup-by-path'; +import type { IResolveContext, WorkspaceLayoutCache } from './WorkspaceLayoutCache'; + +import { normalizeToSlash } from './normalizeSlashes'; + +type ResolveRequest = Parameters[1]; + +/** + * A resolver plugin that optimizes locating the package.json file for a module. + */ +export class KnownDescriptionFilePlugin { + public readonly source: string; + public readonly target: string; + + private readonly _skipForContext: boolean; + private readonly _cache: WorkspaceLayoutCache; + + public constructor(cache: WorkspaceLayoutCache, source: string, target: string, skipForContext?: boolean) { + this.source = source; + this.target = target; + this._cache = cache; + this._skipForContext = !!skipForContext; + } + + public apply(resolver: Resolver): void { + if (this._skipForContext && resolver.options.resolveToContext) { + return; + } + + const target: ReturnType = resolver.ensureHook(this.target); + const { fileSystem } = resolver; + function readDescriptionFileAsJson( + descriptionFilePath: string, + callback: (err: Error | null | undefined, data?: object) => void + ): void { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + fileSystem.readJson!(descriptionFilePath, callback); + } + + function readDescriptionFileWithParse( + descriptionFilePath: string, + callback: (err: Error | null | undefined, data?: object) => void + ): void { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + fileSystem.readFile(descriptionFilePath, (err: Error | null | undefined, data?: string | Buffer) => { + if (!data?.length) { + return callback(err); + } + // eslint-disable-next-line @rushstack/no-new-null + callback(null, JSON.parse(data.toString())); + }); + } + + const readDescriptionFile: ( + descriptionFilePath: string, + cb: (err: Error | null | undefined, data?: object) => void + ) => void = fileSystem.readJson ? readDescriptionFileAsJson : readDescriptionFileWithParse; + + resolver + .getHook(this.source) + .tapAsync(KnownDescriptionFilePlugin.name, (request, resolveContext, callback) => { + const { path } = request; + // No request, nothing to do. + if (!path) return callback(); + + const cache: WorkspaceLayoutCache = this._cache; + + const match: IPrefixMatch | undefined = + cache.contextLookup.findLongestPrefixMatch(path); + // No description file available, proceed without. + if (!match) return callback(); + + const relativePath: string = `.${normalizeToSlash(path.slice(match.index))}`; + const descriptionFileRoot: string = `${path.slice(0, match.index)}`; + const descriptionFilePath: string = `${descriptionFileRoot}${directorySeparator}package.json`; + + const { contextForPackage } = cache; + + readDescriptionFile(descriptionFilePath, (err, descriptionFileData) => { + if (!descriptionFileData) { + resolveContext.missingDependencies?.add(descriptionFilePath); + return callback(err); + } + + resolveContext.fileDependencies?.add(descriptionFilePath); + // Store the resolver context since a WeakMap lookup is cheaper than walking the tree again + contextForPackage.set(descriptionFileData, match.value); + + // Since we don't allow any alternative processing of request, we can mutate it + // instead of cloning it. + request.descriptionFileRoot = descriptionFileRoot; + request.descriptionFilePath = descriptionFilePath; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + request.descriptionFileData = descriptionFileData as any; + request.relativePath = relativePath; + + resolver.doResolve( + target, + request, + 'using description file: ' + descriptionFilePath + ' (relative path: ' + relativePath + ')', + resolveContext, + (e: Error | undefined, result: ResolveRequest | undefined) => { + if (e) return callback(e); + + // Don't allow other processing + // eslint-disable-next-line @rushstack/no-new-null + if (result === undefined) return callback(null, null); + // eslint-disable-next-line @rushstack/no-new-null + callback(null, result); + } + ); + }); + }); + } +} diff --git a/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts new file mode 100644 index 00000000000..84c4b4ed30d --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { sep as directorySeparator } from 'node:path'; + +import type { Resolver } from 'webpack'; +import type { IPrefixMatch } from '@rushstack/lookup-by-path'; +import type { IResolveContext, WorkspaceLayoutCache } from './WorkspaceLayoutCache'; + +import { normalizeToSlash } from './normalizeSlashes'; + +type ResolveRequest = Parameters[1]; + +/** + * A resolver plugin that optimizes resolving installed dependencies for the current package. + * Enforces strict resolution. + */ +export class KnownPackageDependenciesPlugin { + public readonly source: string; + public readonly target: string; + + private readonly _cache: WorkspaceLayoutCache; + + public constructor(cache: WorkspaceLayoutCache, source: string, target: string) { + this.source = source; + this.target = target; + this._cache = cache; + } + + public apply(resolver: Resolver): void { + const target: ReturnType = resolver.ensureHook(this.target); + resolver + .getHook(this.source) + .tapAsync(KnownPackageDependenciesPlugin.name, (request, resolveContext, callback) => { + const { path, request: rawRequest } = request; + if (!path) return callback(); + if (!rawRequest) return callback(); + + const { descriptionFileData } = request; + if (!descriptionFileData) return callback(new Error(`Expected descriptionFileData for ${path}`)); + + const cache: WorkspaceLayoutCache = this._cache; + + const context: IResolveContext | undefined = cache.contextForPackage.get(descriptionFileData); + if (!context) return callback(new Error(`Expected context for ${request.descriptionFileRoot}`)); + + const match: IPrefixMatch | undefined = context.findDependency(rawRequest); + if (!match) return callback(); + + const isPackageRoot: boolean = match.index === rawRequest.length; + const fullySpecified: boolean | undefined = isPackageRoot ? false : request.fullySpecified; + const relativePath: string = isPackageRoot + ? '.' + : `.${normalizeToSlash(rawRequest.slice(match.index))}`; + const { descriptionFileRoot } = match.value; + const obj: ResolveRequest = { + ...request, + path: descriptionFileRoot, + descriptionFileRoot, + descriptionFileData: undefined, + descriptionFilePath: `${descriptionFileRoot}${directorySeparator}package.json`, + + relativePath: relativePath, + request: relativePath, + fullySpecified, + module: false + }; + // eslint-disable-next-line @rushstack/no-new-null + resolver.doResolve(target, obj, null, resolveContext, callback); + }); + } +} diff --git a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts new file mode 100644 index 00000000000..33bc84a199e --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { sep as directorySeparator } from 'node:path'; + +import { LookupByPath, type IPrefixMatch } from '@rushstack/lookup-by-path'; + +import { normalizeToPlatform } from './normalizeSlashes'; + +/** + * Information about a local or installed npm package. + * @beta + */ +export interface ISerializedResolveContext { + /** + * The path to the root folder of this context. + * This path is normalized to use `/` as the separator and should not end with a trailing `/`. + */ + root: string; + /** + * The name of this package. Used to inject a self-reference into the dependency map. + */ + name: string; + /** + * Map of declared dependencies to the ordinal of the corresponding context. + */ + deps: Record; + /** + * Set of relative paths to nested `package.json` files within this context. + * These paths are normalized to use `/` as the separator and should not begin with a leading `./`. + */ + dirInfoFiles?: string[]; +} + +/** + * The serialized form of the cache file. This file is expected to be generated by a separate tool from + * information known to the package manager. Namely, the dependency relationships between packages, and + * all the `package.json` files in the workspace (installed or local). + * @beta + */ +export interface IResolverCacheFile { + /** + * The ordered list of all contexts in the cache + */ + contexts: ISerializedResolveContext[]; +} + +/** + * A context for resolving dependencies in a workspace. + * @beta + */ +export interface IResolveContext { + /** + * The absolute path to the root folder of this context + */ + descriptionFileRoot: string; + /** + * Find the context that corresponds to a module specifier, when requested in the current context. + * @param request - The module specifier to resolve + */ + findDependency(request: string): IPrefixMatch | undefined; +} + +/** + * Options for creating a `WorkspaceLayoutCache`. + * @beta + */ +export interface IWorkspaceLayoutCacheOptions { + /** + * The root folder of the workspace. All paths in the cache file are assumed to be relative to this folder. + */ + workspaceRoot: string; + /** + * The parsed cache data. File reading is left as an exercise for the caller. + */ + cacheData: IResolverCacheFile; +} + +/** + * A cache of workspace layout information. + * @beta + */ +export class WorkspaceLayoutCache { + /** + * A lookup of context roots to their corresponding context objects + */ + public readonly contextLookup: LookupByPath; + /** + * A weak map of package JSON contents to their corresponding context objects + */ + public readonly contextForPackage: WeakMap; + + public constructor(options: IWorkspaceLayoutCacheOptions) { + const { workspaceRoot, cacheData } = options; + + const resolveContexts: IResolveContext[] = []; + const contextLookup: LookupByPath = new LookupByPath(undefined, directorySeparator); + + this.contextLookup = contextLookup; + this.contextForPackage = new WeakMap(); + + // Internal class due to coupling of deserialization. + class ResolveContext implements IResolveContext { + private readonly _serialized: ISerializedResolveContext; + private _descriptionFileRoot: string | undefined; + private _dependencies: LookupByPath | undefined; + + public constructor(serialized: ISerializedResolveContext) { + this._serialized = serialized; + this._descriptionFileRoot = undefined; + this._dependencies = undefined; + } + + public get descriptionFileRoot(): string { + if (!this._descriptionFileRoot) { + this._descriptionFileRoot = `${workspaceRoot}${directorySeparator}${normalizeToPlatform(this._serialized.root)}`; + } + return this._descriptionFileRoot; + } + + public findDependency(request: string): IPrefixMatch | undefined { + if (!this._dependencies) { + // Lazy initialize this object since most packages won't be requested. + const dependencies: LookupByPath = new LookupByPath(undefined, '/'); + // Handle the self-reference scenario + dependencies.setItem(this._serialized.name, this); + for (const [key, ordinal] of Object.entries(this._serialized.deps)) { + dependencies.setItem(key, resolveContexts[ordinal]); + } + this._dependencies = dependencies; + } + + return this._dependencies.findLongestPrefixMatch(request); + } + } + + for (const serialized of cacheData.contexts) { + const resolveContext: IResolveContext = new ResolveContext(serialized); + resolveContexts.push(resolveContext); + + const descriptionFileRoot: string = resolveContext.descriptionFileRoot; + contextLookup.setItem(descriptionFileRoot, resolveContext); + + // Handle nested package.json files. These may modify some properties, but the dependency resolution + // will match the original package root. Typically these are used to set the `type` field to `module`. + if (serialized.dirInfoFiles) { + for (const file of serialized.dirInfoFiles) { + contextLookup.setItemFromSegments( + concat( + // Root is normalized to platform slashes + LookupByPath.iteratePathSegments(descriptionFileRoot, directorySeparator), + // Subpaths are platform-agnostic + LookupByPath.iteratePathSegments(file, '/') + ), + resolveContext + ); + } + } + } + } +} + +function* concat(a: Iterable, b: Iterable): IterableIterator { + yield* a; + yield* b; +} diff --git a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts new file mode 100644 index 00000000000..453556aecef --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { WebpackPluginInstance, Compiler } from 'webpack'; +import type { WorkspaceLayoutCache } from './WorkspaceLayoutCache'; +import { KnownDescriptionFilePlugin } from './KnownDescriptionFilePlugin'; +import { KnownPackageDependenciesPlugin } from './KnownPackageDependenciesPlugin'; + +/** + * Options for constructing a `WorkspaceResolvePlugin`. + * + * @beta + */ +export interface IWorkspaceResolvePluginOptions { + /** + * The cache of workspace layout information. + */ + cache: WorkspaceLayoutCache; +} + +/** + * A Webpack plugin that optimizes package.json lookups and resolution of bare specifiers in a monorepo. + * + * @beta + */ +export class WorkspaceResolvePlugin implements WebpackPluginInstance { + private readonly _cache: WorkspaceLayoutCache; + + public constructor(cache: WorkspaceLayoutCache) { + this._cache = cache; + } + + public apply(compiler: Compiler): void { + compiler.resolverFactory.hooks.resolveOptions + .for('normal') + .tap(WorkspaceResolvePlugin.name, (resolveOptions) => { + // Omit default `node_modules` + if (resolveOptions.modules) { + resolveOptions.modules = resolveOptions.modules.filter((modulePath: string) => { + return modulePath !== 'node_modules'; + }); + } else { + resolveOptions.modules = []; + } + + const cache: WorkspaceLayoutCache = this._cache; + + resolveOptions.plugins ??= []; + resolveOptions.plugins.push( + // Optimize identifying the package.json file for the issuer + new KnownDescriptionFilePlugin(cache, 'parsed-resolve', 'described-resolve'), + // Optimize locating the installed dependencies of the current package + new KnownPackageDependenciesPlugin(cache, 'raw-module', 'resolve-as-module'), + // Optimize loading the package.json file for the destination package + new KnownDescriptionFilePlugin(cache, 'relative', 'described-relative'), + // Optimize locating and loading nested package.json for a directory + new KnownDescriptionFilePlugin(cache, 'undescribed-existing-directory', 'existing-directory', true), + // Optimize locating and loading nested package.json for a file + new KnownDescriptionFilePlugin(cache, 'undescribed-raw-file', 'raw-file') + ); + + return resolveOptions; + }); + } +} diff --git a/webpack/webpack-workspace-resolve-plugin/src/index.ts b/webpack/webpack-workspace-resolve-plugin/src/index.ts new file mode 100644 index 00000000000..8f08f9a77d9 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/index.ts @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +export { WorkspaceResolvePlugin, type IWorkspaceResolvePluginOptions } from './WorkspaceResolvePlugin'; +export { + WorkspaceLayoutCache, + type IWorkspaceLayoutCacheOptions, + type IResolveContext, + type ISerializedResolveContext, + type IResolverCacheFile +} from './WorkspaceLayoutCache'; diff --git a/webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts b/webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts new file mode 100644 index 00000000000..10b3f954688 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { sep } from 'node:path'; + +export const normalizeToSlash: (path: string) => string = + sep === '/' ? (path: string) => path : (path: string) => path.replace(/\\/g, '/'); + +export const normalizeToPlatform: (path: string) => string = + sep === '/' ? (path: string) => path : (path: string) => path.replace(/\//g, sep); diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts b/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts new file mode 100644 index 00000000000..e3b9805835e --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Volume } from 'memfs/lib/volume'; +import type { Compiler, Resolver } from 'webpack'; +import { KnownDescriptionFilePlugin } from '../KnownDescriptionFilePlugin'; +import { WorkspaceLayoutCache } from '../WorkspaceLayoutCache'; + +type ResolveCallback = Parameters[1]; +type ResolveRequest = Parameters[0]; +type ResolveContext = Parameters[1]; +// eslint-disable-next-line @rushstack/no-new-null +type WrappedResolve = ( + request: ResolveRequest, + resolveContext: ResolveContext +) => [Error | false | null | undefined, ResolveRequest | undefined]; + +const parsedJson: Record = { + '/workspace/a/package.json': { name: 'a' }, + '/workspace/a/lib-esm/package.json': { type: 'module' }, + '/workspace/b/package.json': { name: 'b', dependencies: { a: 'workspace:*' } } +}; + +function createResolve(): WrappedResolve { + const fileSystem: Volume = new Volume(); + + const serializedJson: Record = Object.fromEntries( + Object.entries(parsedJson).map(([key, value]) => [key, JSON.stringify(value)]) + ); + + fileSystem.fromJSON(serializedJson); + (fileSystem as Compiler['inputFileSystem']).readJson = ( + path: string, + cb: (err: Error | null | undefined, data?: object) => void + ) => { + const parsed: object | undefined = parsedJson[path]; + if (parsed) { + return cb(null, parsed); + } + return cb(new Error(`No data found for ${path}`)); + }; + + let innerCallback: ResolveCallback | undefined = undefined; + + const resolver: Resolver = { + fileSystem, + doResolve: ( + step: string, + request: ResolveRequest, + message: string, + resolveContext: ResolveContext, + callback: (err: Error | undefined, result: ResolveRequest | undefined) => void + ) => { + return callback(undefined, request); + }, + ensureHook: (step: string) => { + expect(step).toEqual('target'); + }, + getHook: (step: string) => { + expect(step).toEqual('source'); + return { + tapAsync: ( + name: string, + cb: (request: ResolveRequest, resolveContext: ResolveContext, callback: () => void) => void + ) => { + expect(name).toEqual(KnownDescriptionFilePlugin.name); + innerCallback = cb; + } + }; + } + } as unknown as Resolver; + + const cache: WorkspaceLayoutCache = new WorkspaceLayoutCache({ + workspaceRoot: '/workspace', + cacheData: { + contexts: [ + { + root: 'a', + name: 'a', + deps: {}, + dirInfoFiles: ['lib-esm'] + }, + { + root: 'b', + name: 'b', + deps: { a: 0 } + } + ] + } + }); + + const plugin: KnownDescriptionFilePlugin = new KnownDescriptionFilePlugin(cache, 'source', 'target'); + plugin.apply(resolver); + + return ( + request: ResolveRequest, + resolveContext: ResolveContext + ): [Error | false | null | undefined, ResolveRequest | undefined] => { + let result!: [Error | false | null | undefined, ResolveRequest | undefined]; + innerCallback!(request, resolveContext, (( + err: Error | null | false | undefined, + next: ResolveRequest | undefined + ) => { + result = [err, next]; + }) as unknown as Parameters[2]); + return result; + }; +} + +describe(KnownDescriptionFilePlugin.name, () => { + it('should resolve the package.json file for a module', () => { + const resolver: WrappedResolve = createResolve(); + + const fileDependencies: Set = new Set(); + const context: ResolveContext = { fileDependencies }; + + const [err1, result1] = resolver({ path: '/workspace/a/lib/index.js' }, context); + expect(err1).toBeNull(); + expect(result1).toEqual({ + path: '/workspace/a/lib/index.js', + descriptionFileRoot: '/workspace/a', + descriptionFileData: parsedJson['/workspace/a/package.json'], + descriptionFilePath: '/workspace/a/package.json', + relativePath: './lib/index.js' + }); + expect(fileDependencies.size).toEqual(1); + expect(fileDependencies.has('/workspace/a/package.json')).toBeTruthy(); + + fileDependencies.clear(); + + const [err2, result2] = resolver({ path: '/workspace/a/foo/bar/baz.js' }, context); + expect(err2).toBeNull(); + expect(result2).toMatchObject({ + path: '/workspace/a/foo/bar/baz.js', + descriptionFileRoot: '/workspace/a', + descriptionFileData: parsedJson['/workspace/a/package.json'], + descriptionFilePath: '/workspace/a/package.json', + relativePath: './foo/bar/baz.js' + }); + expect(fileDependencies.size).toEqual(1); + expect(fileDependencies.has('/workspace/a/package.json')).toBeTruthy(); + + fileDependencies.clear(); + + const [err3, result3] = resolver({ path: '/workspace/a/lib-esm/index.js' }, context); + expect(err3).toBeNull(); + expect(result3).toMatchObject({ + path: '/workspace/a/lib-esm/index.js', + descriptionFileRoot: '/workspace/a/lib-esm', + descriptionFileData: parsedJson['/workspace/a/lib-esm/package.json'], + descriptionFilePath: '/workspace/a/lib-esm/package.json', + relativePath: './index.js' + }); + expect(fileDependencies.size).toEqual(1); + expect(fileDependencies.has('/workspace/a/lib-esm/package.json')).toBeTruthy(); + + fileDependencies.clear(); + }); + + it('should defer to other plugins if not in a context', () => { + const resolver: WrappedResolve = createResolve(); + + const [err1, result1] = resolver({ path: '/workspace/c/lib/index.js' }, {}); + expect(err1).toBeUndefined(); + expect(result1).toBeUndefined(); + }); +}); diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts b/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts new file mode 100644 index 00000000000..f9408a8cfe9 --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Volume } from 'memfs/lib/volume'; +import type { Compiler, Resolver } from 'webpack'; +import { KnownPackageDependenciesPlugin } from '../KnownPackageDependenciesPlugin'; +import { type IResolveContext, WorkspaceLayoutCache } from '../WorkspaceLayoutCache'; + +type ResolveCallback = Parameters[1]; +type ResolveRequest = Parameters[0]; +type ResolveContext = Parameters[1]; +// eslint-disable-next-line @rushstack/no-new-null +type WrappedResolve = ( + request: ResolveRequest, + resolveContext: ResolveContext +) => [Error | false | null | undefined, ResolveRequest | undefined]; + +const parsedJson: Record = { + '/workspace/a/package.json': { name: 'a' }, + '/workspace/a/lib-esm/package.json': { type: 'module' }, + '/workspace/b/package.json': { name: 'b', dependencies: { a: 'workspace:*' } } +}; + +function createResolve(): WrappedResolve { + const fileSystem: Volume = new Volume(); + + const serializedJson: Record = Object.fromEntries( + Object.entries(parsedJson).map(([key, value]) => [key, JSON.stringify(value)]) + ); + + fileSystem.fromJSON(serializedJson); + (fileSystem as Compiler['inputFileSystem']).readJson = ( + path: string, + cb: (err: Error | null | undefined, data?: object) => void + ) => { + const parsed: object | undefined = parsedJson[path]; + if (parsed) { + return cb(null, parsed); + } + return cb(new Error(`No data found for ${path}`)); + }; + + let innerCallback: ResolveCallback | undefined = undefined; + + const resolver: Resolver = { + fileSystem, + doResolve: ( + step: string, + request: ResolveRequest, + message: string, + resolveContext: ResolveContext, + callback: (err: Error | undefined, result: ResolveRequest | undefined) => void + ) => { + return callback(undefined, request); + }, + ensureHook: (step: string) => { + expect(step).toEqual('target'); + }, + getHook: (step: string) => { + expect(step).toEqual('source'); + return { + tapAsync: ( + name: string, + cb: (request: ResolveRequest, resolveContext: ResolveContext, callback: () => void) => void + ) => { + expect(name).toEqual(KnownPackageDependenciesPlugin.name); + innerCallback = cb; + } + }; + } + } as unknown as Resolver; + + const cache: WorkspaceLayoutCache = new WorkspaceLayoutCache({ + workspaceRoot: '/workspace', + cacheData: { + contexts: [ + { + root: 'a', + name: 'a', + deps: {}, + dirInfoFiles: ['lib-esm'] + }, + { + root: 'b', + name: 'b', + deps: { a: 0 } + } + ] + } + }); + + // Backfill the contexts + for (const [path, json] of Object.entries(parsedJson)) { + const context: IResolveContext | undefined = cache.contextLookup.findChildPath(path); + if (!context) throw new Error(`No context found for ${path}`); + cache.contextForPackage.set(json, context); + } + + const plugin: KnownPackageDependenciesPlugin = new KnownPackageDependenciesPlugin( + cache, + 'source', + 'target' + ); + plugin.apply(resolver); + + return ( + request: ResolveRequest, + resolveContext: ResolveContext + ): [Error | false | null | undefined, ResolveRequest | undefined] => { + let result!: [Error | false | null | undefined, ResolveRequest | undefined]; + innerCallback!(request, resolveContext, (( + err: Error | null | false | undefined, + next: ResolveRequest | undefined + ) => { + result = [err, next]; + }) as unknown as Parameters[2]); + return result; + }; +} + +describe(KnownPackageDependenciesPlugin.name, () => { + it('should find a relevant dependency', () => { + const resolver: WrappedResolve = createResolve(); + + const descriptionFilePath: string = '/workspace/b/package.json'; + const descriptionFileData: object = parsedJson[descriptionFilePath]; + const descriptionFileRoot: string = '/workspace/b'; + + const [err1, result1] = resolver( + { + path: '/workspace/b/lib/foo.js', + request: 'a/lib/index.js', + descriptionFileRoot, + descriptionFileData, + descriptionFilePath, + relativePath: './lib/foo.js' + }, + {} + ); + + expect(err1).toBeFalsy(); + expect(result1).toEqual({ + path: '/workspace/a', + request: './lib/index.js', + descriptionFileRoot: '/workspace/a', + descriptionFilePath: '/workspace/a/package.json', + relativePath: './lib/index.js', + fullySpecified: undefined, + module: false + }); + }); + + it('should handle self-reference', () => { + const resolver: WrappedResolve = createResolve(); + + const descriptionFilePath: string = '/workspace/b/package.json'; + const descriptionFileData: object = parsedJson[descriptionFilePath]; + const descriptionFileRoot: string = '/workspace/b'; + + const [err1, result1] = resolver( + { + path: '/workspace/b/lib/foo.js', + request: 'b/lib/bar.js', + descriptionFileRoot, + descriptionFileData, + descriptionFilePath, + relativePath: './lib/foo.js' + }, + {} + ); + + expect(err1).toBeFalsy(); + expect(result1).toEqual({ + path: '/workspace/b', + request: './lib/bar.js', + descriptionFileRoot: '/workspace/b', + descriptionFilePath: '/workspace/b/package.json', + relativePath: './lib/bar.js', + fullySpecified: undefined, + module: false + }); + }); + + it('should defer to other plugins if not in a context', () => { + const resolver: WrappedResolve = createResolve(); + + const [err1, result1] = resolver({ path: '/workspace/c/lib/index.js' }, {}); + expect(err1).toBeUndefined(); + expect(result1).toBeUndefined(); + }); + + it('should defer to other plugins if the dependency is not found (for fallback)', () => { + const resolver: WrappedResolve = createResolve(); + + const descriptionFilePath: string = '/workspace/a/package.json'; + const descriptionFileData: object = parsedJson[descriptionFilePath]; + const descriptionFileRoot: string = '/workspace/a'; + + const [err1, result1] = resolver( + { + path: '/workspace/a/lib/foo.js', + request: 'events', + descriptionFileRoot, + descriptionFileData, + descriptionFilePath, + relativePath: './lib/foo.js' + }, + {} + ); + + expect(err1).toBeUndefined(); + expect(result1).toBeUndefined(); + }); +}); diff --git a/webpack/webpack-workspace-resolve-plugin/tsconfig.json b/webpack/webpack-workspace-resolve-plugin/tsconfig.json new file mode 100644 index 00000000000..97c868d4e4d --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "./node_modules/local-node-rig/profiles/default/tsconfig-base.json", + "compilerOptions": { + "target": "ES2019", + "types": ["heft-jest", "node"] + } +} From 972d84d8a03519de2c44b41f80e0dd9a9583a5bc Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 12 Aug 2024 21:13:09 +0000 Subject: [PATCH 2/5] Fixup lint --- .../src/test/KnownDescriptionFilePlugin.test.ts | 2 +- .../src/test/KnownPackageDependenciesPlugin.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts b/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts index e3b9805835e..5d94e4781bd 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts @@ -9,10 +9,10 @@ import { WorkspaceLayoutCache } from '../WorkspaceLayoutCache'; type ResolveCallback = Parameters[1]; type ResolveRequest = Parameters[0]; type ResolveContext = Parameters[1]; -// eslint-disable-next-line @rushstack/no-new-null type WrappedResolve = ( request: ResolveRequest, resolveContext: ResolveContext + // eslint-disable-next-line @rushstack/no-new-null ) => [Error | false | null | undefined, ResolveRequest | undefined]; const parsedJson: Record = { diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts b/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts index f9408a8cfe9..f03fd51a25e 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts @@ -9,10 +9,10 @@ import { type IResolveContext, WorkspaceLayoutCache } from '../WorkspaceLayoutCa type ResolveCallback = Parameters[1]; type ResolveRequest = Parameters[0]; type ResolveContext = Parameters[1]; -// eslint-disable-next-line @rushstack/no-new-null type WrappedResolve = ( request: ResolveRequest, resolveContext: ResolveContext + // eslint-disable-next-line @rushstack/no-new-null ) => [Error | false | null | undefined, ResolveRequest | undefined]; const parsedJson: Record = { From d1986b378ce75b01149061ac8c2241d55e3ef867 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 12 Aug 2024 21:42:27 +0000 Subject: [PATCH 3/5] Update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 942474aba62..1fcfb338a58 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/webpack/set-webpack-public-path-plugin](./webpack/set-webpack-public-path-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fset-webpack-public-path-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fset-webpack-public-path-plugin) | [changelog](./webpack/set-webpack-public-path-plugin/CHANGELOG.md) | [@rushstack/set-webpack-public-path-plugin](https://www.npmjs.com/package/@rushstack/set-webpack-public-path-plugin) | | [/webpack/webpack-embedded-dependencies-plugin](./webpack/webpack-embedded-dependencies-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fwebpack-embedded-dependencies-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fwebpack-embedded-dependencies-plugin) | [changelog](./webpack/webpack-embedded-dependencies-plugin/CHANGELOG.md) | [@rushstack/webpack-embedded-dependencies-plugin](https://www.npmjs.com/package/@rushstack/webpack-embedded-dependencies-plugin) | | [/webpack/webpack-plugin-utilities](./webpack/webpack-plugin-utilities/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fwebpack-plugin-utilities.svg)](https://badge.fury.io/js/%40rushstack%2Fwebpack-plugin-utilities) | [changelog](./webpack/webpack-plugin-utilities/CHANGELOG.md) | [@rushstack/webpack-plugin-utilities](https://www.npmjs.com/package/@rushstack/webpack-plugin-utilities) | +| [/webpack/webpack-workspace-resolve-plugin](./webpack/webpack-workspace-resolve-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fwebpack-workspace-resolve-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fwebpack-workspace-resolve-plugin) | [changelog](./webpack/webpack-workspace-resolve-plugin/CHANGELOG.md) | [@rushstack/webpack-workspace-resolve-plugin](https://www.npmjs.com/package/@rushstack/webpack-workspace-resolve-plugin) | | [/webpack/webpack4-localization-plugin](./webpack/webpack4-localization-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fwebpack4-localization-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fwebpack4-localization-plugin) | [changelog](./webpack/webpack4-localization-plugin/CHANGELOG.md) | [@rushstack/webpack4-localization-plugin](https://www.npmjs.com/package/@rushstack/webpack4-localization-plugin) | | [/webpack/webpack4-module-minifier-plugin](./webpack/webpack4-module-minifier-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fwebpack4-module-minifier-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fwebpack4-module-minifier-plugin) | [changelog](./webpack/webpack4-module-minifier-plugin/CHANGELOG.md) | [@rushstack/webpack4-module-minifier-plugin](https://www.npmjs.com/package/@rushstack/webpack4-module-minifier-plugin) | | [/webpack/webpack5-load-themed-styles-loader](./webpack/webpack5-load-themed-styles-loader/) | [![npm version](https://badge.fury.io/js/%40microsoft%2Fwebpack5-load-themed-styles-loader.svg)](https://badge.fury.io/js/%40microsoft%2Fwebpack5-load-themed-styles-loader) | [changelog](./webpack/webpack5-load-themed-styles-loader/CHANGELOG.md) | [@microsoft/webpack5-load-themed-styles-loader](https://www.npmjs.com/package/@microsoft/webpack5-load-themed-styles-loader) | From b921f1a2e894cb179c8a482806cac503c0ffe746 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 12 Aug 2024 22:52:34 +0000 Subject: [PATCH 4/5] Fix cross-platform --- .../webpack-workspace-resolve-plugin.api.md | 7 + .../src/KnownDescriptionFilePlugin.ts | 8 +- .../src/KnownPackageDependenciesPlugin.ts | 8 +- .../src/WorkspaceLayoutCache.ts | 44 ++++- .../src/normalizeSlashes.ts | 10 -- .../test/KnownDescriptionFilePlugin.test.ts | 169 +++++++----------- .../KnownPackageDependenciesPlugin.test.ts | 163 +++++------------ .../src/test/createResolveForTests.ts | 121 +++++++++++++ 8 files changed, 279 insertions(+), 251 deletions(-) delete mode 100644 webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts create mode 100644 webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts diff --git a/common/reviews/api/webpack-workspace-resolve-plugin.api.md b/common/reviews/api/webpack-workspace-resolve-plugin.api.md index c47850e2591..9b812e3a32e 100644 --- a/common/reviews/api/webpack-workspace-resolve-plugin.api.md +++ b/common/reviews/api/webpack-workspace-resolve-plugin.api.md @@ -31,6 +31,7 @@ export interface ISerializedResolveContext { // @beta export interface IWorkspaceLayoutCacheOptions { cacheData: IResolverCacheFile; + resolverPathSeparator?: '/' | '\\'; workspaceRoot: string; } @@ -44,6 +45,12 @@ export class WorkspaceLayoutCache { constructor(options: IWorkspaceLayoutCacheOptions); readonly contextForPackage: WeakMap; readonly contextLookup: LookupByPath; + // (undocumented) + readonly normalizeToPlatform: (input: string) => string; + // (undocumented) + readonly normalizeToSlash: (input: string) => string; + // (undocumented) + readonly resolverPathSeparator: string; } // @beta diff --git a/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts index 6bbdc299857..a4b54bf565b 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts @@ -1,14 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { sep as directorySeparator } from 'node:path'; - import type { Resolver } from 'webpack'; import type { IPrefixMatch } from '@rushstack/lookup-by-path'; import type { IResolveContext, WorkspaceLayoutCache } from './WorkspaceLayoutCache'; -import { normalizeToSlash } from './normalizeSlashes'; - type ResolveRequest = Parameters[1]; /** @@ -76,9 +72,9 @@ export class KnownDescriptionFilePlugin { // No description file available, proceed without. if (!match) return callback(); - const relativePath: string = `.${normalizeToSlash(path.slice(match.index))}`; + const relativePath: string = `.${cache.normalizeToSlash(path.slice(match.index))}`; const descriptionFileRoot: string = `${path.slice(0, match.index)}`; - const descriptionFilePath: string = `${descriptionFileRoot}${directorySeparator}package.json`; + const descriptionFilePath: string = `${descriptionFileRoot}${cache.resolverPathSeparator}package.json`; const { contextForPackage } = cache; diff --git a/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts index 84c4b4ed30d..7cf4b504c50 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts @@ -1,14 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { sep as directorySeparator } from 'node:path'; - import type { Resolver } from 'webpack'; import type { IPrefixMatch } from '@rushstack/lookup-by-path'; import type { IResolveContext, WorkspaceLayoutCache } from './WorkspaceLayoutCache'; -import { normalizeToSlash } from './normalizeSlashes'; - type ResolveRequest = Parameters[1]; /** @@ -51,14 +47,14 @@ export class KnownPackageDependenciesPlugin { const fullySpecified: boolean | undefined = isPackageRoot ? false : request.fullySpecified; const relativePath: string = isPackageRoot ? '.' - : `.${normalizeToSlash(rawRequest.slice(match.index))}`; + : `.${cache.normalizeToSlash(rawRequest.slice(match.index))}`; const { descriptionFileRoot } = match.value; const obj: ResolveRequest = { ...request, path: descriptionFileRoot, descriptionFileRoot, descriptionFileData: undefined, - descriptionFilePath: `${descriptionFileRoot}${directorySeparator}package.json`, + descriptionFilePath: `${descriptionFileRoot}${cache.resolverPathSeparator}package.json`, relativePath: relativePath, request: relativePath, diff --git a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts index 33bc84a199e..6fc8bb25ad1 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts @@ -5,8 +5,6 @@ import { sep as directorySeparator } from 'node:path'; import { LookupByPath, type IPrefixMatch } from '@rushstack/lookup-by-path'; -import { normalizeToPlatform } from './normalizeSlashes'; - /** * Information about a local or installed npm package. * @beta @@ -74,6 +72,23 @@ export interface IWorkspaceLayoutCacheOptions { * The parsed cache data. File reading is left as an exercise for the caller. */ cacheData: IResolverCacheFile; + /** + * The directory separator used in the `path` field of the resolver inputs. + * Will usually be `path.sep`. + */ + resolverPathSeparator?: '/' | '\\'; +} + +function preservePath(path: string): string { + return path; +} + +function backslashToSlash(path: string): string { + return path.replace(/\\/g, '/'); +} + +function slashToBackslash(path: string): string { + return path.replace(/\//g, '\\'); } /** @@ -90,15 +105,32 @@ export class WorkspaceLayoutCache { */ public readonly contextForPackage: WeakMap; + public readonly resolverPathSeparator: string; + public readonly normalizeToSlash: (input: string) => string; + public readonly normalizeToPlatform: (input: string) => string; + public constructor(options: IWorkspaceLayoutCacheOptions) { - const { workspaceRoot, cacheData } = options; + const { workspaceRoot, cacheData, resolverPathSeparator = directorySeparator } = options; + + if (resolverPathSeparator !== '/' && resolverPathSeparator !== '\\') { + throw new Error(`Unsupported directory separator: ${resolverPathSeparator}`); + } const resolveContexts: IResolveContext[] = []; - const contextLookup: LookupByPath = new LookupByPath(undefined, directorySeparator); + const contextLookup: LookupByPath = new LookupByPath(undefined, resolverPathSeparator); this.contextLookup = contextLookup; this.contextForPackage = new WeakMap(); + const normalizeToSlash: (path: string) => string = + resolverPathSeparator === '/' ? preservePath : backslashToSlash; + const normalizeToPlatform: (path: string) => string = + resolverPathSeparator === '/' ? preservePath : slashToBackslash; + + this.resolverPathSeparator = resolverPathSeparator; + this.normalizeToSlash = normalizeToSlash; + this.normalizeToPlatform = normalizeToPlatform; + // Internal class due to coupling of deserialization. class ResolveContext implements IResolveContext { private readonly _serialized: ISerializedResolveContext; @@ -113,7 +145,7 @@ export class WorkspaceLayoutCache { public get descriptionFileRoot(): string { if (!this._descriptionFileRoot) { - this._descriptionFileRoot = `${workspaceRoot}${directorySeparator}${normalizeToPlatform(this._serialized.root)}`; + this._descriptionFileRoot = `${workspaceRoot}${resolverPathSeparator}${normalizeToPlatform(this._serialized.root)}`; } return this._descriptionFileRoot; } @@ -148,7 +180,7 @@ export class WorkspaceLayoutCache { contextLookup.setItemFromSegments( concat( // Root is normalized to platform slashes - LookupByPath.iteratePathSegments(descriptionFileRoot, directorySeparator), + LookupByPath.iteratePathSegments(descriptionFileRoot, resolverPathSeparator), // Subpaths are platform-agnostic LookupByPath.iteratePathSegments(file, '/') ), diff --git a/webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts b/webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts deleted file mode 100644 index 10b3f954688..00000000000 --- a/webpack/webpack-workspace-resolve-plugin/src/normalizeSlashes.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { sep } from 'node:path'; - -export const normalizeToSlash: (path: string) => string = - sep === '/' ? (path: string) => path : (path: string) => path.replace(/\\/g, '/'); - -export const normalizeToPlatform: (path: string) => string = - sep === '/' ? (path: string) => path : (path: string) => path.replace(/\//g, sep); diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts b/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts index 5d94e4781bd..78cc2153941 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/test/KnownDescriptionFilePlugin.test.ts @@ -1,115 +1,25 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { Volume } from 'memfs/lib/volume'; -import type { Compiler, Resolver } from 'webpack'; import { KnownDescriptionFilePlugin } from '../KnownDescriptionFilePlugin'; -import { WorkspaceLayoutCache } from '../WorkspaceLayoutCache'; - -type ResolveCallback = Parameters[1]; -type ResolveRequest = Parameters[0]; -type ResolveContext = Parameters[1]; -type WrappedResolve = ( - request: ResolveRequest, - resolveContext: ResolveContext - // eslint-disable-next-line @rushstack/no-new-null -) => [Error | false | null | undefined, ResolveRequest | undefined]; - -const parsedJson: Record = { - '/workspace/a/package.json': { name: 'a' }, - '/workspace/a/lib-esm/package.json': { type: 'module' }, - '/workspace/b/package.json': { name: 'b', dependencies: { a: 'workspace:*' } } -}; - -function createResolve(): WrappedResolve { - const fileSystem: Volume = new Volume(); - - const serializedJson: Record = Object.fromEntries( - Object.entries(parsedJson).map(([key, value]) => [key, JSON.stringify(value)]) - ); - - fileSystem.fromJSON(serializedJson); - (fileSystem as Compiler['inputFileSystem']).readJson = ( - path: string, - cb: (err: Error | null | undefined, data?: object) => void - ) => { - const parsed: object | undefined = parsedJson[path]; - if (parsed) { - return cb(null, parsed); - } - return cb(new Error(`No data found for ${path}`)); - }; - - let innerCallback: ResolveCallback | undefined = undefined; - - const resolver: Resolver = { - fileSystem, - doResolve: ( - step: string, - request: ResolveRequest, - message: string, - resolveContext: ResolveContext, - callback: (err: Error | undefined, result: ResolveRequest | undefined) => void - ) => { - return callback(undefined, request); - }, - ensureHook: (step: string) => { - expect(step).toEqual('target'); - }, - getHook: (step: string) => { - expect(step).toEqual('source'); - return { - tapAsync: ( - name: string, - cb: (request: ResolveRequest, resolveContext: ResolveContext, callback: () => void) => void - ) => { - expect(name).toEqual(KnownDescriptionFilePlugin.name); - innerCallback = cb; - } - }; - } - } as unknown as Resolver; - - const cache: WorkspaceLayoutCache = new WorkspaceLayoutCache({ - workspaceRoot: '/workspace', - cacheData: { - contexts: [ - { - root: 'a', - name: 'a', - deps: {}, - dirInfoFiles: ['lib-esm'] - }, - { - root: 'b', - name: 'b', - deps: { a: 0 } - } - ] - } - }); - const plugin: KnownDescriptionFilePlugin = new KnownDescriptionFilePlugin(cache, 'source', 'target'); - plugin.apply(resolver); - - return ( - request: ResolveRequest, - resolveContext: ResolveContext - ): [Error | false | null | undefined, ResolveRequest | undefined] => { - let result!: [Error | false | null | undefined, ResolveRequest | undefined]; - innerCallback!(request, resolveContext, (( - err: Error | null | false | undefined, - next: ResolveRequest | undefined - ) => { - result = [err, next]; - }) as unknown as Parameters[2]); - return result; - }; +import { + parsedJson, + createResolveForTests, + type WrappedResolve, + type ResolveContext +} from './createResolveForTests'; + +function createResolve(separator: '/' | '\\'): WrappedResolve { + return createResolveForTests(separator, (cache, resolver) => { + const plugin: KnownDescriptionFilePlugin = new KnownDescriptionFilePlugin(cache, 'source', 'target'); + plugin.apply(resolver); + }); } describe(KnownDescriptionFilePlugin.name, () => { - it('should resolve the package.json file for a module', () => { - const resolver: WrappedResolve = createResolve(); + it('should resolve the package.json file for a module (/)', () => { + const resolver: WrappedResolve = createResolve('/'); const fileDependencies: Set = new Set(); const context: ResolveContext = { fileDependencies }; @@ -157,8 +67,57 @@ describe(KnownDescriptionFilePlugin.name, () => { fileDependencies.clear(); }); + it('should resolve the package.json file for a module (\\)', () => { + const resolver: WrappedResolve = createResolve('\\'); + + const fileDependencies: Set = new Set(); + const context: ResolveContext = { fileDependencies }; + + const [err1, result1] = resolver({ path: '\\workspace\\a\\lib\\index.js' }, context); + expect(err1).toBeNull(); + expect(result1).toEqual({ + path: '\\workspace\\a\\lib\\index.js', + descriptionFileRoot: '\\workspace\\a', + descriptionFileData: parsedJson['/workspace/a/package.json'], + descriptionFilePath: '\\workspace\\a\\package.json', + relativePath: './lib/index.js' + }); + expect(fileDependencies.size).toEqual(1); + expect(fileDependencies.has('\\workspace\\a\\package.json')).toBeTruthy(); + + fileDependencies.clear(); + + const [err2, result2] = resolver({ path: '\\workspace\\a\\foo\\bar\\baz.js' }, context); + expect(err2).toBeNull(); + expect(result2).toMatchObject({ + path: '\\workspace\\a\\foo\\bar\\baz.js', + descriptionFileRoot: '\\workspace\\a', + descriptionFileData: parsedJson['/workspace/a/package.json'], + descriptionFilePath: '\\workspace\\a\\package.json', + relativePath: './foo/bar/baz.js' + }); + expect(fileDependencies.size).toEqual(1); + expect(fileDependencies.has('\\workspace\\a\\package.json')).toBeTruthy(); + + fileDependencies.clear(); + + const [err3, result3] = resolver({ path: '\\workspace\\a\\lib-esm\\index.js' }, context); + expect(err3).toBeNull(); + expect(result3).toMatchObject({ + path: '\\workspace\\a\\lib-esm\\index.js', + descriptionFileRoot: '\\workspace\\a\\lib-esm', + descriptionFileData: parsedJson['/workspace/a/lib-esm/package.json'], + descriptionFilePath: '\\workspace\\a\\lib-esm\\package.json', + relativePath: './index.js' + }); + expect(fileDependencies.size).toEqual(1); + expect(fileDependencies.has('\\workspace\\a\\lib-esm\\package.json')).toBeTruthy(); + + fileDependencies.clear(); + }); + it('should defer to other plugins if not in a context', () => { - const resolver: WrappedResolve = createResolve(); + const resolver: WrappedResolve = createResolve('/'); const [err1, result1] = resolver({ path: '/workspace/c/lib/index.js' }, {}); expect(err1).toBeUndefined(); diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts b/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts index f03fd51a25e..16bf41647e3 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/test/KnownPackageDependenciesPlugin.test.ts @@ -1,126 +1,23 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { Volume } from 'memfs/lib/volume'; -import type { Compiler, Resolver } from 'webpack'; import { KnownPackageDependenciesPlugin } from '../KnownPackageDependenciesPlugin'; -import { type IResolveContext, WorkspaceLayoutCache } from '../WorkspaceLayoutCache'; - -type ResolveCallback = Parameters[1]; -type ResolveRequest = Parameters[0]; -type ResolveContext = Parameters[1]; -type WrappedResolve = ( - request: ResolveRequest, - resolveContext: ResolveContext - // eslint-disable-next-line @rushstack/no-new-null -) => [Error | false | null | undefined, ResolveRequest | undefined]; - -const parsedJson: Record = { - '/workspace/a/package.json': { name: 'a' }, - '/workspace/a/lib-esm/package.json': { type: 'module' }, - '/workspace/b/package.json': { name: 'b', dependencies: { a: 'workspace:*' } } -}; - -function createResolve(): WrappedResolve { - const fileSystem: Volume = new Volume(); - - const serializedJson: Record = Object.fromEntries( - Object.entries(parsedJson).map(([key, value]) => [key, JSON.stringify(value)]) - ); - - fileSystem.fromJSON(serializedJson); - (fileSystem as Compiler['inputFileSystem']).readJson = ( - path: string, - cb: (err: Error | null | undefined, data?: object) => void - ) => { - const parsed: object | undefined = parsedJson[path]; - if (parsed) { - return cb(null, parsed); - } - return cb(new Error(`No data found for ${path}`)); - }; - - let innerCallback: ResolveCallback | undefined = undefined; - - const resolver: Resolver = { - fileSystem, - doResolve: ( - step: string, - request: ResolveRequest, - message: string, - resolveContext: ResolveContext, - callback: (err: Error | undefined, result: ResolveRequest | undefined) => void - ) => { - return callback(undefined, request); - }, - ensureHook: (step: string) => { - expect(step).toEqual('target'); - }, - getHook: (step: string) => { - expect(step).toEqual('source'); - return { - tapAsync: ( - name: string, - cb: (request: ResolveRequest, resolveContext: ResolveContext, callback: () => void) => void - ) => { - expect(name).toEqual(KnownPackageDependenciesPlugin.name); - innerCallback = cb; - } - }; - } - } as unknown as Resolver; - - const cache: WorkspaceLayoutCache = new WorkspaceLayoutCache({ - workspaceRoot: '/workspace', - cacheData: { - contexts: [ - { - root: 'a', - name: 'a', - deps: {}, - dirInfoFiles: ['lib-esm'] - }, - { - root: 'b', - name: 'b', - deps: { a: 0 } - } - ] - } +import { createResolveForTests, parsedJson, type WrappedResolve } from './createResolveForTests'; + +function createResolve(separator: '/' | '\\'): WrappedResolve { + return createResolveForTests(separator, (cache, resolver) => { + const plugin: KnownPackageDependenciesPlugin = new KnownPackageDependenciesPlugin( + cache, + 'source', + 'target' + ); + plugin.apply(resolver); }); - - // Backfill the contexts - for (const [path, json] of Object.entries(parsedJson)) { - const context: IResolveContext | undefined = cache.contextLookup.findChildPath(path); - if (!context) throw new Error(`No context found for ${path}`); - cache.contextForPackage.set(json, context); - } - - const plugin: KnownPackageDependenciesPlugin = new KnownPackageDependenciesPlugin( - cache, - 'source', - 'target' - ); - plugin.apply(resolver); - - return ( - request: ResolveRequest, - resolveContext: ResolveContext - ): [Error | false | null | undefined, ResolveRequest | undefined] => { - let result!: [Error | false | null | undefined, ResolveRequest | undefined]; - innerCallback!(request, resolveContext, (( - err: Error | null | false | undefined, - next: ResolveRequest | undefined - ) => { - result = [err, next]; - }) as unknown as Parameters[2]); - return result; - }; } describe(KnownPackageDependenciesPlugin.name, () => { - it('should find a relevant dependency', () => { - const resolver: WrappedResolve = createResolve(); + it('should find a relevant dependency (/)', () => { + const resolver: WrappedResolve = createResolve('/'); const descriptionFilePath: string = '/workspace/b/package.json'; const descriptionFileData: object = parsedJson[descriptionFilePath]; @@ -149,9 +46,39 @@ describe(KnownPackageDependenciesPlugin.name, () => { module: false }); }); + it('should find a relevant dependency (\\)', () => { + const resolver: WrappedResolve = createResolve('\\'); + + const descriptionFilePath: string = '\\workspace\\b\\package.json'; + const descriptionFileData: object = parsedJson['/workspace/b/package.json']; + const descriptionFileRoot: string = '\\workspace\\b'; + + const [err1, result1] = resolver( + { + path: '\\workspace\\b\\lib\\foo.js', + request: 'a/lib/index.js', + descriptionFileRoot, + descriptionFileData, + descriptionFilePath, + relativePath: './lib/foo.js' + }, + {} + ); + + expect(err1).toBeFalsy(); + expect(result1).toEqual({ + path: '\\workspace\\a', + request: './lib/index.js', + descriptionFileRoot: '\\workspace\\a', + descriptionFilePath: '\\workspace\\a\\package.json', + relativePath: './lib/index.js', + fullySpecified: undefined, + module: false + }); + }); it('should handle self-reference', () => { - const resolver: WrappedResolve = createResolve(); + const resolver: WrappedResolve = createResolve('/'); const descriptionFilePath: string = '/workspace/b/package.json'; const descriptionFileData: object = parsedJson[descriptionFilePath]; @@ -182,7 +109,7 @@ describe(KnownPackageDependenciesPlugin.name, () => { }); it('should defer to other plugins if not in a context', () => { - const resolver: WrappedResolve = createResolve(); + const resolver: WrappedResolve = createResolve('/'); const [err1, result1] = resolver({ path: '/workspace/c/lib/index.js' }, {}); expect(err1).toBeUndefined(); @@ -190,7 +117,7 @@ describe(KnownPackageDependenciesPlugin.name, () => { }); it('should defer to other plugins if the dependency is not found (for fallback)', () => { - const resolver: WrappedResolve = createResolve(); + const resolver: WrappedResolve = createResolve('/'); const descriptionFilePath: string = '/workspace/a/package.json'; const descriptionFileData: object = parsedJson[descriptionFilePath]; diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts b/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts new file mode 100644 index 00000000000..e2dc0e9308f --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Volume } from 'memfs/lib/volume'; +import type { Compiler, Resolver } from 'webpack'; + +import { WorkspaceLayoutCache, type IResolveContext } from '../WorkspaceLayoutCache'; + +export type ResolveCallback = Parameters[1]; +export type ResolveRequest = Parameters[0]; +export type ResolveContext = Parameters[1]; +export type WrappedResolve = ( + request: ResolveRequest, + resolveContext: ResolveContext + // eslint-disable-next-line @rushstack/no-new-null +) => [Error | false | null | undefined, ResolveRequest | undefined]; + +export const parsedJson: Record = { + '/workspace/a/package.json': { name: 'a' }, + '/workspace/a/lib-esm/package.json': { type: 'module' }, + '/workspace/b/package.json': { name: 'b', dependencies: { a: 'workspace:*' } } +}; + +export function createResolveForTests( + separator: '/' | '\\', + attachPlugins: (cache: WorkspaceLayoutCache, resolver: Resolver) => void +): WrappedResolve { + const fileSystem: Volume = new Volume(); + + const cache: WorkspaceLayoutCache = new WorkspaceLayoutCache({ + workspaceRoot: `${separator}workspace`, + cacheData: { + contexts: [ + { + root: 'a', + name: 'a', + deps: {}, + dirInfoFiles: ['lib-esm'] + }, + { + root: 'b', + name: 'b', + deps: { a: 0 } + } + ] + }, + resolverPathSeparator: separator + }); + + const platformJson: Record = Object.fromEntries( + Object.entries(parsedJson).map(([key, value]) => [cache.normalizeToPlatform(key), value]) + ); + + const serializedJson: Record = Object.fromEntries( + Object.entries(platformJson).map(([key, value]) => [key, JSON.stringify(value)]) + ); + + fileSystem.fromJSON(serializedJson); + (fileSystem as Compiler['inputFileSystem']).readJson = ( + path: string, + cb: (err: Error | null | undefined, data?: object) => void + ) => { + const parsed: object | undefined = platformJson[path]; + if (parsed) { + return cb(null, parsed); + } + return cb(new Error(`No data found for ${path}`)); + }; + + let innerCallback: ResolveCallback | undefined = undefined; + + const resolver: Resolver = { + fileSystem, + doResolve: ( + step: string, + request: ResolveRequest, + message: string, + resolveContext: ResolveContext, + callback: (err: Error | undefined, result: ResolveRequest | undefined) => void + ) => { + return callback(undefined, request); + }, + ensureHook: (step: string) => { + expect(step).toEqual('target'); + }, + getHook: (step: string) => { + expect(step).toEqual('source'); + return { + tapAsync: ( + name: string, + cb: (request: ResolveRequest, resolveContext: ResolveContext, callback: () => void) => void + ) => { + innerCallback = cb; + } + }; + } + } as unknown as Resolver; + + // Backfill the contexts + for (const [path, json] of Object.entries(platformJson)) { + const context: IResolveContext | undefined = cache.contextLookup.findChildPath(path); + if (!context) throw new Error(`No context found for ${path}`); + cache.contextForPackage.set(json, context); + } + + attachPlugins(cache, resolver); + + return ( + request: ResolveRequest, + resolveContext: ResolveContext + ): [Error | false | null | undefined, ResolveRequest | undefined] => { + let result!: [Error | false | null | undefined, ResolveRequest | undefined]; + innerCallback!(request, resolveContext, (( + err: Error | null | false | undefined, + next: ResolveRequest | undefined + ) => { + result = [err, next]; + }) as unknown as Parameters[2]); + return result; + }; +} From ff833cebcb3e4fa3e0bf07738762051fd3f9faf3 Mon Sep 17 00:00:00 2001 From: David Michon Date: Tue, 13 Aug 2024 00:53:25 +0000 Subject: [PATCH 5/5] Address PR feedback --- .../config/subspaces/default/pnpm-lock.yaml | 6 --- .../webpack-workspace-resolve-plugin.api.md | 7 ++- .../README.md | 2 +- .../package.json | 4 +- .../src/KnownDescriptionFilePlugin.ts | 48 +++++++++++-------- .../src/KnownPackageDependenciesPlugin.ts | 36 ++++++++++---- .../src/WorkspaceLayoutCache.ts | 29 ++++++----- .../src/WorkspaceResolvePlugin.ts | 1 + .../src/index.ts | 1 + .../src/test/createResolveForTests.ts | 2 +- 10 files changed, 85 insertions(+), 51 deletions(-) diff --git a/common/config/subspaces/default/pnpm-lock.yaml b/common/config/subspaces/default/pnpm-lock.yaml index 48d0cb9879f..55afe1fd331 100644 --- a/common/config/subspaces/default/pnpm-lock.yaml +++ b/common/config/subspaces/default/pnpm-lock.yaml @@ -4406,12 +4406,6 @@ importers: '@rushstack/lookup-by-path': specifier: workspace:* version: link:../../libraries/lookup-by-path - '@types/tapable': - specifier: 1.0.6 - version: 1.0.6 - tapable: - specifier: 2.2.1 - version: 2.2.1 devDependencies: '@rushstack/heft': specifier: workspace:* diff --git a/common/reviews/api/webpack-workspace-resolve-plugin.api.md b/common/reviews/api/webpack-workspace-resolve-plugin.api.md index 9b812e3a32e..909e04e02bc 100644 --- a/common/reviews/api/webpack-workspace-resolve-plugin.api.md +++ b/common/reviews/api/webpack-workspace-resolve-plugin.api.md @@ -9,6 +9,9 @@ import { IPrefixMatch } from '@rushstack/lookup-by-path'; import { LookupByPath } from '@rushstack/lookup-by-path'; import type { WebpackPluginInstance } from 'webpack'; +// @beta +export type IPathNormalizationFunction = ((input: string) => string) | undefined; + // @beta export interface IResolveContext { descriptionFileRoot: string; @@ -46,9 +49,9 @@ export class WorkspaceLayoutCache { readonly contextForPackage: WeakMap; readonly contextLookup: LookupByPath; // (undocumented) - readonly normalizeToPlatform: (input: string) => string; + readonly normalizeToPlatform: IPathNormalizationFunction; // (undocumented) - readonly normalizeToSlash: (input: string) => string; + readonly normalizeToSlash: IPathNormalizationFunction; // (undocumented) readonly resolverPathSeparator: string; } diff --git a/webpack/webpack-workspace-resolve-plugin/README.md b/webpack/webpack-workspace-resolve-plugin/README.md index 7397a279f45..eff328065f1 100644 --- a/webpack/webpack-workspace-resolve-plugin/README.md +++ b/webpack/webpack-workspace-resolve-plugin/README.md @@ -19,7 +19,7 @@ When using this plugin, the following options should be configured for your reso ## Limitations -This plugin depends on the presence of a cache file in the workspace to function. Data in this cache file is assumed not to change while the webpack process is running, although the file will be. +This plugin depends on the presence of a cache file in the workspace to function. Data in this cache file is assumed not to change while the webpack process is running. **Note:** Generating the cache file is not in the scope of this plugin. diff --git a/webpack/webpack-workspace-resolve-plugin/package.json b/webpack/webpack-workspace-resolve-plugin/package.json index 8c521454c80..02296e1343f 100644 --- a/webpack/webpack-workspace-resolve-plugin/package.json +++ b/webpack/webpack-workspace-resolve-plugin/package.json @@ -23,9 +23,7 @@ "@types/node": "*" }, "dependencies": { - "@rushstack/lookup-by-path": "workspace:*", - "@types/tapable": "1.0.6", - "tapable": "2.2.1" + "@rushstack/lookup-by-path": "workspace:*" }, "devDependencies": { "@rushstack/heft": "workspace:*", diff --git a/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts index a4b54bf565b..769721c181f 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/KnownDescriptionFilePlugin.ts @@ -9,6 +9,8 @@ type ResolveRequest = Parameters[1]; /** * A resolver plugin that optimizes locating the package.json file for a module. + * + * @internal */ export class KnownDescriptionFilePlugin { public readonly source: string; @@ -17,6 +19,13 @@ export class KnownDescriptionFilePlugin { private readonly _skipForContext: boolean; private readonly _cache: WorkspaceLayoutCache; + /** + * Constructs a new instance of `KnownDescriptionFilePlugin`. + * @param cache - The workspace layout cache + * @param source - The resolve step to hook into + * @param target - The resolve step to delegate to + * @param skipForContext - If true, don't apply this plugin if the resolver is configured to resolve to a context + */ public constructor(cache: WorkspaceLayoutCache, source: string, target: string, skipForContext?: boolean) { this.source = source; this.target = target; @@ -31,19 +40,11 @@ export class KnownDescriptionFilePlugin { const target: ReturnType = resolver.ensureHook(this.target); const { fileSystem } = resolver; - function readDescriptionFileAsJson( - descriptionFilePath: string, - callback: (err: Error | null | undefined, data?: object) => void - ): void { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - fileSystem.readJson!(descriptionFilePath, callback); - } function readDescriptionFileWithParse( descriptionFilePath: string, callback: (err: Error | null | undefined, data?: object) => void ): void { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion fileSystem.readFile(descriptionFilePath, (err: Error | null | undefined, data?: string | Buffer) => { if (!data?.length) { return callback(err); @@ -56,23 +57,28 @@ export class KnownDescriptionFilePlugin { const readDescriptionFile: ( descriptionFilePath: string, cb: (err: Error | null | undefined, data?: object) => void - ) => void = fileSystem.readJson ? readDescriptionFileAsJson : readDescriptionFileWithParse; + ) => void = fileSystem.readJson?.bind(fileSystem) ?? readDescriptionFileWithParse; resolver .getHook(this.source) .tapAsync(KnownDescriptionFilePlugin.name, (request, resolveContext, callback) => { const { path } = request; - // No request, nothing to do. - if (!path) return callback(); + if (!path) { + // No request, nothing to do. + return callback(); + } const cache: WorkspaceLayoutCache = this._cache; const match: IPrefixMatch | undefined = cache.contextLookup.findLongestPrefixMatch(path); - // No description file available, proceed without. - if (!match) return callback(); + if (!match) { + // No description file available, proceed without. + return callback(); + } - const relativePath: string = `.${cache.normalizeToSlash(path.slice(match.index))}`; + const remainingPath: string = path.slice(match.index); + const relativePath: string = `.${cache.normalizeToSlash?.(remainingPath) ?? remainingPath}`; const descriptionFileRoot: string = `${path.slice(0, match.index)}`; const descriptionFilePath: string = `${descriptionFileRoot}${cache.resolverPathSeparator}package.json`; @@ -92,21 +98,25 @@ export class KnownDescriptionFilePlugin { // instead of cloning it. request.descriptionFileRoot = descriptionFileRoot; request.descriptionFilePath = descriptionFilePath; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - request.descriptionFileData = descriptionFileData as any; + request.descriptionFileData = descriptionFileData; request.relativePath = relativePath; + // Delegate to the resolver step at `target`. resolver.doResolve( target, request, 'using description file: ' + descriptionFilePath + ' (relative path: ' + relativePath + ')', resolveContext, (e: Error | undefined, result: ResolveRequest | undefined) => { - if (e) return callback(e); + if (e) { + return callback(e); + } // Don't allow other processing - // eslint-disable-next-line @rushstack/no-new-null - if (result === undefined) return callback(null, null); + if (result === undefined) { + // eslint-disable-next-line @rushstack/no-new-null + return callback(null, null); + } // eslint-disable-next-line @rushstack/no-new-null callback(null, result); } diff --git a/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts index 7cf4b504c50..8e03064bfcf 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/KnownPackageDependenciesPlugin.ts @@ -10,6 +10,8 @@ type ResolveRequest = Parameters[1]; /** * A resolver plugin that optimizes resolving installed dependencies for the current package. * Enforces strict resolution. + * + * @internal */ export class KnownPackageDependenciesPlugin { public readonly source: string; @@ -17,6 +19,12 @@ export class KnownPackageDependenciesPlugin { private readonly _cache: WorkspaceLayoutCache; + /** + * Constructs a new instance of `KnownPackageDependenciesPlugin`. + * @param cache - The workspace layout cache + * @param source - The resolve step to hook into + * @param target - The resolve step to delegate to + */ public constructor(cache: WorkspaceLayoutCache, source: string, target: string) { this.source = source; this.target = target; @@ -25,29 +33,41 @@ export class KnownPackageDependenciesPlugin { public apply(resolver: Resolver): void { const target: ReturnType = resolver.ensureHook(this.target); + resolver .getHook(this.source) .tapAsync(KnownPackageDependenciesPlugin.name, (request, resolveContext, callback) => { const { path, request: rawRequest } = request; - if (!path) return callback(); - if (!rawRequest) return callback(); + if (!path) { + return callback(); + } + + if (!rawRequest) { + return callback(); + } const { descriptionFileData } = request; - if (!descriptionFileData) return callback(new Error(`Expected descriptionFileData for ${path}`)); + if (!descriptionFileData) { + return callback(new Error(`Expected descriptionFileData for ${path}`)); + } const cache: WorkspaceLayoutCache = this._cache; const context: IResolveContext | undefined = cache.contextForPackage.get(descriptionFileData); - if (!context) return callback(new Error(`Expected context for ${request.descriptionFileRoot}`)); + if (!context) { + return callback(new Error(`Expected context for ${request.descriptionFileRoot}`)); + } const match: IPrefixMatch | undefined = context.findDependency(rawRequest); - if (!match) return callback(); + if (!match) { + return callback(); + } const isPackageRoot: boolean = match.index === rawRequest.length; const fullySpecified: boolean | undefined = isPackageRoot ? false : request.fullySpecified; - const relativePath: string = isPackageRoot - ? '.' - : `.${cache.normalizeToSlash(rawRequest.slice(match.index))}`; + const remainingPath: string = isPackageRoot ? '.' : `.${rawRequest.slice(match.index)}`; + const relativePath: string = + (remainingPath.length > 1 && cache.normalizeToSlash?.(remainingPath)) || remainingPath; const { descriptionFileRoot } = match.value; const obj: ResolveRequest = { ...request, diff --git a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts index 6fc8bb25ad1..e3296428d9a 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts @@ -79,9 +79,13 @@ export interface IWorkspaceLayoutCacheOptions { resolverPathSeparator?: '/' | '\\'; } -function preservePath(path: string): string { - return path; -} +/** + * A function that normalizes a path to a platform-specific format (if needed). + * Will be undefined if the platform uses `/` as the path separator. + * + * @beta + */ +export type IPathNormalizationFunction = ((input: string) => string) | undefined; function backslashToSlash(path: string): string { return path.replace(/\\/g, '/'); @@ -106,8 +110,8 @@ export class WorkspaceLayoutCache { public readonly contextForPackage: WeakMap; public readonly resolverPathSeparator: string; - public readonly normalizeToSlash: (input: string) => string; - public readonly normalizeToPlatform: (input: string) => string; + public readonly normalizeToSlash: IPathNormalizationFunction; + public readonly normalizeToPlatform: IPathNormalizationFunction; public constructor(options: IWorkspaceLayoutCacheOptions) { const { workspaceRoot, cacheData, resolverPathSeparator = directorySeparator } = options; @@ -122,16 +126,16 @@ export class WorkspaceLayoutCache { this.contextLookup = contextLookup; this.contextForPackage = new WeakMap(); - const normalizeToSlash: (path: string) => string = - resolverPathSeparator === '/' ? preservePath : backslashToSlash; - const normalizeToPlatform: (path: string) => string = - resolverPathSeparator === '/' ? preservePath : slashToBackslash; + const normalizeToSlash: IPathNormalizationFunction = + resolverPathSeparator === '\\' ? backslashToSlash : undefined; + const normalizeToPlatform: IPathNormalizationFunction = + resolverPathSeparator === '\\' ? slashToBackslash : undefined; this.resolverPathSeparator = resolverPathSeparator; this.normalizeToSlash = normalizeToSlash; this.normalizeToPlatform = normalizeToPlatform; - // Internal class due to coupling of deserialization. + // Internal class due to coupling to `resolveContexts` class ResolveContext implements IResolveContext { private readonly _serialized: ISerializedResolveContext; private _descriptionFileRoot: string | undefined; @@ -145,7 +149,9 @@ export class WorkspaceLayoutCache { public get descriptionFileRoot(): string { if (!this._descriptionFileRoot) { - this._descriptionFileRoot = `${workspaceRoot}${resolverPathSeparator}${normalizeToPlatform(this._serialized.root)}`; + this._descriptionFileRoot = `${workspaceRoot}${resolverPathSeparator}${ + normalizeToPlatform?.(this._serialized.root) ?? this._serialized.root + }`; } return this._descriptionFileRoot; } @@ -157,6 +163,7 @@ export class WorkspaceLayoutCache { // Handle the self-reference scenario dependencies.setItem(this._serialized.name, this); for (const [key, ordinal] of Object.entries(this._serialized.deps)) { + // This calls into the array of instances that is owned by WorkpaceLayoutCache dependencies.setItem(key, resolveContexts[ordinal]); } this._dependencies = dependencies; diff --git a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts index 453556aecef..b6564e5824d 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceResolvePlugin.ts @@ -2,6 +2,7 @@ // See LICENSE in the project root for license information. import type { WebpackPluginInstance, Compiler } from 'webpack'; + import type { WorkspaceLayoutCache } from './WorkspaceLayoutCache'; import { KnownDescriptionFilePlugin } from './KnownDescriptionFilePlugin'; import { KnownPackageDependenciesPlugin } from './KnownPackageDependenciesPlugin'; diff --git a/webpack/webpack-workspace-resolve-plugin/src/index.ts b/webpack/webpack-workspace-resolve-plugin/src/index.ts index 8f08f9a77d9..61866c3a162 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/index.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/index.ts @@ -4,6 +4,7 @@ export { WorkspaceResolvePlugin, type IWorkspaceResolvePluginOptions } from './WorkspaceResolvePlugin'; export { WorkspaceLayoutCache, + type IPathNormalizationFunction, type IWorkspaceLayoutCacheOptions, type IResolveContext, type ISerializedResolveContext, diff --git a/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts b/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts index e2dc0e9308f..0fd8670ad12 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/test/createResolveForTests.ts @@ -48,7 +48,7 @@ export function createResolveForTests( }); const platformJson: Record = Object.fromEntries( - Object.entries(parsedJson).map(([key, value]) => [cache.normalizeToPlatform(key), value]) + Object.entries(parsedJson).map(([key, value]) => [cache.normalizeToPlatform?.(key) ?? key, value]) ); const serializedJson: Record = Object.fromEntries(