From 0e809a0a4ec89c6e74d9cffb34a8dbe1d1e97b29 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 14 Sep 2024 07:24:49 +0000 Subject: [PATCH 1/8] Add file paths to extractor-metadata.json --- common/reviews/api/package-extractor.api.md | 1 + .../package-extractor/src/AssetHandler.ts | 141 ++++++ .../package-extractor/src/PackageExtractor.ts | 214 ++++----- .../src/test/PackageExtractor.test.ts | 26 +- .../PackageExtractor.test.ts.snap | 451 ++++++++++++++++++ 5 files changed, 717 insertions(+), 116 deletions(-) create mode 100644 libraries/package-extractor/src/AssetHandler.ts create mode 100644 libraries/package-extractor/src/test/__snapshots__/PackageExtractor.test.ts.snap diff --git a/common/reviews/api/package-extractor.api.md b/common/reviews/api/package-extractor.api.md index a3cd76854ab..b756b381184 100644 --- a/common/reviews/api/package-extractor.api.md +++ b/common/reviews/api/package-extractor.api.md @@ -17,6 +17,7 @@ export interface IExtractorDependencyConfiguration { // @public export interface IExtractorMetadataJson { + files: string[]; links: ILinkInfo[]; mainProjectName: string; projects: IProjectInfoJson[]; diff --git a/libraries/package-extractor/src/AssetHandler.ts b/libraries/package-extractor/src/AssetHandler.ts new file mode 100644 index 00000000000..c1374a839cb --- /dev/null +++ b/libraries/package-extractor/src/AssetHandler.ts @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import path from 'node:path'; +import fs from 'node:fs'; +import { FileSystem, Path, type FileSystemStats } from '@rushstack/node-core-library'; +import type { ITerminal } from '@rushstack/terminal'; +import { ArchiveManager } from './ArchiveManager'; + +export interface IAssetHandlerOptions { + terminal: ITerminal; + targetRootFolder: string; + createArchiveOnly?: boolean; + archiveFilePath?: string; +} + +export interface IIncludeAssetOptions { + sourceFilePath?: string; + sourceFileStats?: FileSystemStats; + sourceFileContent?: string | Buffer; + targetFilePath: string; + ignoreIfExisting?: boolean; +} + +export interface IIncludeAssetPathOptions extends IIncludeAssetOptions { + sourceFilePath: string; + sourceFileContent?: never; +} + +export interface IIncludeExistingAssetPathOptions extends IIncludeAssetOptions { + sourceFilePath?: never; + sourceFileContent?: never; +} + +export interface IIncludeAssetContentOptions extends IIncludeAssetOptions { + sourceFileContent: string | Buffer; + sourceFilePath?: never; + sourceFileStats?: never; +} + +export class AssetHandler { + private readonly _terminal: ITerminal; + private readonly _targetRootFolder: string; + private readonly _createArchiveOnly: boolean; + private readonly _archiveManager: ArchiveManager | undefined; + private readonly _archiveFilePath: string | undefined; + private readonly _includedAssetPaths: Set = new Set(); + private _isFinalized: boolean = false; + + public constructor(options: IAssetHandlerOptions) { + const { terminal, targetRootFolder, archiveFilePath, createArchiveOnly = false } = options; + this._terminal = terminal; + this._targetRootFolder = targetRootFolder; + this._archiveFilePath = archiveFilePath; + this._createArchiveOnly = createArchiveOnly; + if (this._archiveFilePath) { + this._archiveManager = new ArchiveManager(); + } + } + + public async includeAssetAsync(options: IIncludeAssetPathOptions): Promise; + public async includeAssetAsync(options: IIncludeExistingAssetPathOptions): Promise; + public async includeAssetAsync(options: IIncludeAssetContentOptions): Promise; + public async includeAssetAsync(options: IIncludeAssetOptions): Promise { + const { sourceFileContent, targetFilePath, ignoreIfExisting = false } = options; + let { sourceFilePath } = options; + + if (this._isFinalized) { + throw new Error('includeAssetAsync() cannot be called after finalizeAsync()'); + } + if (!sourceFilePath && !sourceFileContent) { + if (!Path.isUnder(targetFilePath, this._targetRootFolder)) { + throw new Error('The existing asset path must be under the target root folder'); + } + sourceFilePath = targetFilePath; + } + if (sourceFilePath && sourceFileContent) { + throw new Error('Either sourceFilePath or sourceFileContent must be provided, but not both'); + } + if (this._includedAssetPaths.has(targetFilePath)) { + if (ignoreIfExisting) { + return; + } + throw new Error(`The asset at path "${targetFilePath}" has already been included`); + } + + if (!this._createArchiveOnly) { + // Ignore when the source file is the same as the target file, as it's a no-op + if (sourceFilePath && sourceFilePath !== targetFilePath) { + // Use the fs.copyFile API instead of FileSystem.copyFileAsync() since copyFileAsync performs + // a needless stat() call to determine if it's a file or folder, and we already know it's a file. + try { + await fs.promises.copyFile(sourceFilePath, targetFilePath, fs.constants.COPYFILE_EXCL); + } catch (e: unknown) { + if (!FileSystem.isNotExistError(e as Error)) { + throw e; + } + // The parent folder may not exist, so ensure it exists before trying to copy again + await FileSystem.ensureFolderAsync(path.dirname(targetFilePath)); + await fs.promises.copyFile(sourceFilePath, targetFilePath, fs.constants.COPYFILE_EXCL); + } + } else if (sourceFileContent) { + await FileSystem.writeFileAsync(targetFilePath, sourceFileContent, { + ensureFolderExists: true + }); + } + } + + if (this._archiveManager) { + const targetRelativeFilePath: string = path.relative(this._targetRootFolder, targetFilePath); + if (sourceFilePath) { + await this._archiveManager.addToArchiveAsync({ + filePath: sourceFilePath, + archivePath: targetRelativeFilePath + }); + } else if (sourceFileContent) { + await this._archiveManager.addToArchiveAsync({ + fileData: sourceFileContent, + archivePath: targetRelativeFilePath + }); + } + } + + this._includedAssetPaths.add(targetFilePath); + } + + public get assetPaths(): string[] { + return [...this._includedAssetPaths]; + } + + public async finalizeAsync(): Promise { + if (this._isFinalized) { + throw new Error('finalizeAsync() has already been called'); + } + if (this._archiveManager && this._archiveFilePath) { + this._terminal.writeLine(`Creating archive at "${this._archiveFilePath}"`); + await this._archiveManager.createArchiveAsync(this._archiveFilePath); + } + this._isFinalized = true; + } +} diff --git a/libraries/package-extractor/src/PackageExtractor.ts b/libraries/package-extractor/src/PackageExtractor.ts index d425ef8a42f..07c89b0eb44 100644 --- a/libraries/package-extractor/src/PackageExtractor.ts +++ b/libraries/package-extractor/src/PackageExtractor.ts @@ -2,7 +2,6 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import * as fs from 'fs'; import { type IMinimatch, Minimatch } from 'minimatch'; import semver from 'semver'; import npmPacklist from 'npm-packlist'; @@ -19,8 +18,8 @@ import { } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; -import { ArchiveManager } from './ArchiveManager'; import { SymlinkAnalyzer, type ILinkInfo, type PathNode } from './SymlinkAnalyzer'; +import { AssetHandler } from './AssetHandler'; import { matchesWithStar } from './Utils'; import { createLinksScriptFilename, scriptsFolderPath } from './PathConstants'; @@ -72,6 +71,10 @@ export interface IExtractorMetadataJson { * A list of all links that are part of the extracted project. */ links: ILinkInfo[]; + /** + * A list of all files that are part of the extracted project. + */ + files: string[]; } /** @@ -103,7 +106,7 @@ interface IExtractorState { projectConfigurationsByName: Map; dependencyConfigurationsByName: Map; symlinkAnalyzer: SymlinkAnalyzer; - archiver?: ArchiveManager; + assetHandler: AssetHandler; } /** @@ -342,7 +345,6 @@ export class PackageExtractor { } } - let archiver: ArchiveManager | undefined; let archiveFilePath: string | undefined; if (createArchiveFilePath) { if (path.extname(createArchiveFilePath) !== '.zip') { @@ -350,7 +352,6 @@ export class PackageExtractor { } archiveFilePath = path.resolve(targetRootFolder, createArchiveFilePath); - archiver = new ArchiveManager(); } await FileSystem.ensureFolderAsync(targetRootFolder); @@ -386,8 +387,14 @@ export class PackageExtractor { projectConfigurationsByPath: new Map(projectConfigurations.map((p) => [p.projectFolder, p])), dependencyConfigurationsByName: new Map(), symlinkAnalyzer: new SymlinkAnalyzer({ requiredSourceParentPath: sourceRootFolder }), - archiver + assetHandler: new AssetHandler({ + terminal, + targetRootFolder, + createArchiveOnly, + archiveFilePath + }) }; + // set state dependencyConfigurationsByName for (const dependencyConfiguration of dependencyConfigurations || []) { const { dependencyName } = dependencyConfiguration; @@ -401,10 +408,7 @@ export class PackageExtractor { } await this._performExtractionAsync(options, state); - if (archiver && archiveFilePath) { - terminal.writeLine(`Creating archive at "${archiveFilePath}"`); - await archiver.createArchiveAsync(archiveFilePath); - } + await state.assetHandler.finalizeAsync(); } private static _normalizeOptions(options: IExtractorOptions): IExtractorOptions { @@ -448,7 +452,7 @@ export class PackageExtractor { linkCreationScriptPath, createArchiveOnly } = options; - const { projectConfigurationsByName, foldersToCopy, symlinkAnalyzer } = state; + const { projectConfigurationsByName, foldersToCopy, assetHandler, symlinkAnalyzer } = state; const mainProjectConfiguration: IExtractorProjectConfiguration | undefined = projectConfigurationsByName.get(mainProjectName); @@ -507,22 +511,18 @@ export class PackageExtractor { case 'script': { terminal.writeLine(`Creating ${createLinksScriptFilename}`); const createLinksSourceFilePath: string = `${scriptsFolderPath}/${createLinksScriptFilename}`; - const createLinksTargetFilePath: string = linkCreationScriptPath - ? path.resolve(targetRootFolder, linkCreationScriptPath) - : `${targetRootFolder}/${createLinksScriptFilename}`; + const createLinksTargetFilePath: string = path.resolve( + targetRootFolder, + linkCreationScriptPath || createLinksScriptFilename + ); let createLinksScriptContent: string = await FileSystem.readFileAsync(createLinksSourceFilePath); createLinksScriptContent = createLinksScriptContent.replace( TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, Path.convertToSlashes(path.relative(path.dirname(createLinksTargetFilePath), targetRootFolder)) ); - if (!createArchiveOnly) { - await FileSystem.writeFileAsync(createLinksTargetFilePath, createLinksScriptContent, { - ensureFolderExists: true - }); - } - await state.archiver?.addToArchiveAsync({ - fileData: createLinksScriptContent, - archivePath: path.relative(targetRootFolder, createLinksTargetFilePath) + await assetHandler.includeAssetAsync({ + sourceFileContent: createLinksScriptContent, + targetFilePath: createLinksTargetFilePath }); break; } @@ -739,7 +739,7 @@ export class PackageExtractor { * Example input: "C:\\MyRepo\\libraries\\my-lib" * Example output: "C:\\MyRepo\\common\\deploy\\libraries\\my-lib" */ - private _remapPathForExtractorFolder( + private _remapSourcePathForExtractorFolder( absolutePathInSourceFolder: string, options: IExtractorOptions ): string { @@ -758,14 +758,32 @@ export class PackageExtractor { * Example input: "C:\\MyRepo\\libraries\\my-lib" * Example output: "libraries/my-lib" */ - private _remapPathForExtractorMetadata( + private _remapSourcePathForExtractorMetadata( absolutePathInSourceFolder: string, options: IExtractorOptions ): string { const { sourceRootFolder } = options; - const relativePath: string = path.relative(sourceRootFolder, absolutePathInSourceFolder); + return this._remapPathForExtractorMetadata(sourceRootFolder, absolutePathInSourceFolder); + } + + /** + * Maps a file path from IExtractorOptions.targetRootFolder to relative path + * + * Example input: "C:\\MyRepo\\libraries\\my-lib" + * Example output: "libraries/my-lib" + */ + private _remapTargetPathForExtractorMetadata( + absolutePathInTargetFolder: string, + options: IExtractorOptions + ): string { + const { targetRootFolder } = options; + return this._remapPathForExtractorMetadata(targetRootFolder, absolutePathInTargetFolder); + } + + private _remapPathForExtractorMetadata(folderPath: string, filePath: string): string { + const relativePath: string = path.relative(folderPath, filePath); if (relativePath.startsWith('..')) { - throw new Error(`Source path "${absolutePathInSourceFolder}" is not under "${sourceRootFolder}"`); + throw new Error(`Path "${filePath}" is not under "${folderPath}"`); } return Path.convertToSlashes(relativePath); } @@ -778,8 +796,8 @@ export class PackageExtractor { options: IExtractorOptions, state: IExtractorState ): Promise { - const { includeNpmIgnoreFiles, targetRootFolder } = options; - const { projectConfigurationsByPath, packageJsonByPath, dependencyConfigurationsByName, archiver } = + const { includeNpmIgnoreFiles } = options; + const { projectConfigurationsByPath, packageJsonByPath, dependencyConfigurationsByName, assetHandler } = state; let useNpmIgnoreFilter: boolean = false; @@ -788,8 +806,8 @@ export class PackageExtractor { projectConfigurationsByPath.get(sourceFolderRealPath); const packagesJson: IPackageJson | undefined = packageJsonByPath.get(sourceFolderRealPath); - // As this function will be used to copy folder for both project inside monorepo and third party dependencies insides node_modules - // Third party dependencies won't have project configurations + // As this function will be used to copy folder for both project inside monorepo and third party + // dependencies insides node_modules. Third party dependencies won't have project configurations const isLocalProject: boolean = !!sourceProjectConfiguration; // Function to filter files inside local project or third party dependencies. @@ -849,13 +867,9 @@ export class PackageExtractor { useNpmIgnoreFilter = true; } - const targetFolderPath: string = this._remapPathForExtractorFolder(sourceFolderPath, options); - + const targetFolderPath: string = this._remapSourcePathForExtractorFolder(sourceFolderPath, options); if (useNpmIgnoreFilter) { const npmPackFiles: string[] = await PackageExtractor.getPackageIncludedFilesAsync(sourceFolderPath); - - const alreadyCopiedSourcePaths: Set = new Set(); - await Async.forEachAsync( npmPackFiles, async (npmPackFile: string) => { @@ -871,35 +885,17 @@ export class PackageExtractor { return; } - // We can detect the duplicates by comparing the path.resolve() result. - const copySourcePath: string = path.resolve(sourceFolderPath, npmPackFile); - - if (alreadyCopiedSourcePaths.has(copySourcePath)) { - return; - } - alreadyCopiedSourcePaths.add(copySourcePath); - - const copyDestinationPath: string = path.join(targetFolderPath, npmPackFile); - - const copySourcePathNode: PathNode = await state.symlinkAnalyzer.analyzePathAsync({ - inputPath: copySourcePath + const sourceFilePath: string = path.resolve(sourceFolderPath, npmPackFile); + const { kind, linkStats: sourceFileStats } = await state.symlinkAnalyzer.analyzePathAsync({ + inputPath: sourceFilePath }); - if (copySourcePathNode.kind !== 'link') { - if (!options.createArchiveOnly) { - await FileSystem.ensureFolderAsync(path.dirname(copyDestinationPath)); - // Use the fs.copyFile API instead of FileSystem.copyFileAsync() since copyFileAsync performs - // a needless stat() call to determine if it's a file or folder, and we already know it's a file. - await fs.promises.copyFile(copySourcePath, copyDestinationPath, fs.constants.COPYFILE_EXCL); - } - - if (archiver) { - const archivePath: string = path.relative(targetRootFolder, copyDestinationPath); - await archiver.addToArchiveAsync({ - filePath: copySourcePath, - archivePath, - stats: copySourcePathNode.linkStats - }); - } + if (kind !== 'link') { + const targetFilePath: string = path.resolve(targetFolderPath, npmPackFile); + await assetHandler.includeAssetAsync({ + sourceFilePath, + sourceFileStats, + targetFilePath + }); } }, { @@ -955,23 +951,12 @@ export class PackageExtractor { return; } - const targetPath: string = path.join(targetFolderPath, relativeSourcePath); - if (!options.createArchiveOnly) { - // Manually call fs.copyFile to avoid unnecessary stat calls. - const targetParentPath: string = path.dirname(targetPath); - await FileSystem.ensureFolderAsync(targetParentPath); - await fs.promises.copyFile(sourcePath, targetPath, fs.constants.COPYFILE_EXCL); - } - - // Add the file to the archive. Only need to add files since directories will be auto-created - if (archiver) { - const archivePath: string = path.relative(targetRootFolder, targetPath); - await archiver.addToArchiveAsync({ - filePath: sourcePath, - archivePath: archivePath, - stats: sourcePathNode.linkStats - }); - } + const targetFilePath: string = path.resolve(targetFolderPath, relativeSourcePath); + await assetHandler.includeAssetAsync({ + sourceFilePath: sourcePath, + sourceFileStats: sourcePathNode.linkStats, + targetFilePath + }); } else if (sourcePathNode.kind === 'folder') { const children: string[] = await FileSystem.readFolderItemNamesAsync(sourcePath); for (const child of children) { @@ -996,47 +981,46 @@ export class PackageExtractor { options: IExtractorOptions, state: IExtractorState ): Promise { - const linkInfo: ILinkInfo = { + const { kind, linkPath, targetPath } = { kind: originalLinkInfo.kind, - linkPath: this._remapPathForExtractorFolder(originalLinkInfo.linkPath, options), - targetPath: this._remapPathForExtractorFolder(originalLinkInfo.targetPath, options) + linkPath: this._remapSourcePathForExtractorFolder(originalLinkInfo.linkPath, options), + targetPath: this._remapSourcePathForExtractorFolder(originalLinkInfo.targetPath, options) }; - const newLinkFolder: string = path.dirname(linkInfo.linkPath); + const newLinkFolder: string = path.dirname(linkPath); await FileSystem.ensureFolderAsync(newLinkFolder); // Link to the relative path for symlinks - const relativeTargetPath: string = path.relative(newLinkFolder, linkInfo.targetPath); + const relativeTargetPath: string = path.relative(newLinkFolder, targetPath); // NOTE: This logic is based on NpmLinkManager._createSymlink() - if (linkInfo.kind === 'fileLink') { + if (kind === 'fileLink') { // For files, we use a Windows "hard link", because creating a symbolic link requires // administrator permission. However hard links seem to cause build failures on Mac, // so for all other operating systems we use symbolic links for this case. if (process.platform === 'win32') { await FileSystem.createHardLinkAsync({ linkTargetPath: relativeTargetPath, - newLinkPath: linkInfo.linkPath + newLinkPath: linkPath }); } else { await FileSystem.createSymbolicLinkFileAsync({ linkTargetPath: relativeTargetPath, - newLinkPath: linkInfo.linkPath + newLinkPath: linkPath }); } } else { // Junctions are only supported on Windows. This will create a symbolic link on other platforms. await FileSystem.createSymbolicLinkJunctionAsync({ linkTargetPath: relativeTargetPath, - newLinkPath: linkInfo.linkPath + newLinkPath: linkPath }); } // Since the created symlinks have the required relative paths, they can be added directly to // the archive. - await state.archiver?.addToArchiveAsync({ - filePath: linkInfo.linkPath, - archivePath: path.relative(options.targetRootFolder, linkInfo.linkPath) + await state.assetHandler.includeAssetAsync({ + targetFilePath: linkPath }); } @@ -1062,37 +1046,37 @@ export class PackageExtractor { const extractorMetadataJson: IExtractorMetadataJson = { mainProjectName, projects: [], - links: [] + links: [], + files: [] }; for (const { projectFolder, projectName } of projectConfigurationsByPath.values()) { if (state.foldersToCopy.has(projectFolder)) { extractorMetadataJson.projects.push({ projectName, - path: this._remapPathForExtractorMetadata(projectFolder, options) + path: this._remapSourcePathForExtractorMetadata(projectFolder, options) }); } } // Remap the links to be relative to target folder - for (const absoluteLinkInfo of state.symlinkAnalyzer.reportSymlinks()) { - const relativeInfo: ILinkInfo = { - kind: absoluteLinkInfo.kind, - linkPath: this._remapPathForExtractorMetadata(absoluteLinkInfo.linkPath, options), - targetPath: this._remapPathForExtractorMetadata(absoluteLinkInfo.targetPath, options) - }; - extractorMetadataJson.links.push(relativeInfo); + for (const { kind, linkPath, targetPath } of state.symlinkAnalyzer.reportSymlinks()) { + extractorMetadataJson.links.push({ + kind, + linkPath: this._remapSourcePathForExtractorMetadata(linkPath, options), + targetPath: this._remapSourcePathForExtractorMetadata(targetPath, options) + }); } - const extractorMetadataFileContent: string = JSON.stringify(extractorMetadataJson, undefined, 0); - if (!options.createArchiveOnly) { - await FileSystem.writeFileAsync(extractorMetadataFilePath, extractorMetadataFileContent, { - ensureFolderExists: true - }); + for (const assetPath of state.assetHandler.assetPaths) { + extractorMetadataJson.files.push(this._remapTargetPathForExtractorMetadata(assetPath, options)); } - const archivePath: string = path.relative(targetRootFolder, extractorMetadataFilePath); - await state.archiver?.addToArchiveAsync({ archivePath, fileData: extractorMetadataFileContent }); + const extractorMetadataFileContent: string = JSON.stringify(extractorMetadataJson, undefined, 0); + await state.assetHandler.includeAssetAsync({ + sourceFileContent: extractorMetadataFileContent, + targetFilePath: extractorMetadataFilePath + }); } private async _makeBinLinksAsync(options: IExtractorOptions, state: IExtractorState): Promise { @@ -1105,7 +1089,10 @@ export class PackageExtractor { await Async.forEachAsync( extractedProjectFolders, async (projectFolder: string) => { - const extractedProjectFolder: string = this._remapPathForExtractorFolder(projectFolder, options); + const extractedProjectFolder: string = this._remapSourcePathForExtractorFolder( + projectFolder, + options + ); const extractedProjectNodeModulesFolder: string = path.join(extractedProjectFolder, 'node_modules'); const extractedProjectBinFolder: string = path.join(extractedProjectNodeModulesFolder, '.bin'); @@ -1117,14 +1104,13 @@ export class PackageExtractor { } ); - if (linkedBinPackageNames.length && state.archiver) { + if (linkedBinPackageNames.length) { const binFolderItems: string[] = await FileSystem.readFolderItemNamesAsync(extractedProjectBinFolder); for (const binFolderItem of binFolderItems) { - const binFilePath: string = path.join(extractedProjectBinFolder, binFolderItem); - await state.archiver.addToArchiveAsync({ - filePath: binFilePath, - archivePath: path.relative(options.targetRootFolder, binFilePath) + const binFilePath: string = path.resolve(extractedProjectBinFolder, binFolderItem); + await state.assetHandler.includeAssetAsync({ + targetFilePath: binFilePath }); } } diff --git a/libraries/package-extractor/src/test/PackageExtractor.test.ts b/libraries/package-extractor/src/test/PackageExtractor.test.ts index 06bdf4ca67a..02ed545242a 100644 --- a/libraries/package-extractor/src/test/PackageExtractor.test.ts +++ b/libraries/package-extractor/src/test/PackageExtractor.test.ts @@ -4,9 +4,13 @@ import path from 'path'; import type { ChildProcess } from 'child_process'; -import { Executable, FileSystem } from '@rushstack/node-core-library'; +import { Executable, FileSystem, Sort } from '@rushstack/node-core-library'; import { Terminal, StringBufferTerminalProvider } from '@rushstack/terminal'; -import { PackageExtractor, type IExtractorProjectConfiguration } from '../PackageExtractor'; +import { + PackageExtractor, + type IExtractorProjectConfiguration, + type IExtractorMetadataJson +} from '../PackageExtractor'; // Do this work in the "temp/test.jest" directory since it gets cleaned on clean runs const extractorTargetFolder: string = path.resolve(__dirname, '..', '..', 'test-output'); @@ -529,6 +533,15 @@ describe(PackageExtractor.name, () => { ) ) ).resolves.toEqual(path.join(targetFolder, project3RelativePath, 'src', 'index.js')); + + const metadataFileContent: string = await FileSystem.readFileAsync( + `${targetFolder}/extractor-metadata.json` + ); + const metadata: IExtractorMetadataJson = JSON.parse(metadataFileContent); + Sort.sortBy(metadata.files, (x) => x); + Sort.sortBy(metadata.links, (x) => x.linkPath); + Sort.sortBy(metadata.projects, (x) => x.path); + expect(metadata).toMatchSnapshot(); }); it('should extract project with script linkCreation and custom linkCreationScriptPath', async () => { @@ -596,5 +609,14 @@ describe(PackageExtractor.name, () => { ) ) ).resolves.toEqual(path.join(targetFolder, project3RelativePath, 'src', 'index.js')); + + const metadataFileContent: string = await FileSystem.readFileAsync( + `${path.dirname(linkCreationScriptPath)}/extractor-metadata.json` + ); + const metadata: IExtractorMetadataJson = JSON.parse(metadataFileContent); + Sort.sortBy(metadata.files, (x) => x); + Sort.sortBy(metadata.links, (x) => x.linkPath); + Sort.sortBy(metadata.projects, (x) => x.path); + expect(metadata).toMatchSnapshot(); }); }); diff --git a/libraries/package-extractor/src/test/__snapshots__/PackageExtractor.test.ts.snap b/libraries/package-extractor/src/test/__snapshots__/PackageExtractor.test.ts.snap new file mode 100644 index 00000000000..3647b02eb2f --- /dev/null +++ b/libraries/package-extractor/src/test/__snapshots__/PackageExtractor.test.ts.snap @@ -0,0 +1,451 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`PackageExtractor should extract project with script linkCreation 1`] = ` +Object { + "files": Array [ + "build-tests/package-extractor-test-01/.rush/temp/shrinkwrap-deps.json", + "build-tests/package-extractor-test-01/package.json", + "build-tests/package-extractor-test-01/src/index.js", + "build-tests/package-extractor-test-01/src/subdir/file.js", + "build-tests/package-extractor-test-02/.rush/temp/shrinkwrap-deps.json", + "build-tests/package-extractor-test-02/package.json", + "build-tests/package-extractor-test-02/src/index.js", + "build-tests/package-extractor-test-03/.rush/temp/shrinkwrap-deps.json", + "build-tests/package-extractor-test-03/package.json", + "build-tests/package-extractor-test-03/src/index.js", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/LICENSE", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/README.md", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/assert.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/assert/strict.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/async_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/buffer.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/child_process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/cluster.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/console.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/constants.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/crypto.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/dgram.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/diagnostics_channel.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/dns.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/dns/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/domain.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/fs.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/fs/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/globals.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/globals.global.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/http.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/http2.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/https.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/index.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/inspector.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/module.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/net.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/os.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/package.json", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/path.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/perf_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/punycode.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/querystring.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/readline.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/repl.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream/consumers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream/web.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/string_decoder.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/timers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/timers/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/tls.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/trace_events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/tty.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/url.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/util.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/v8.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/vm.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/wasi.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/worker_threads.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/zlib.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/LICENSE", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/README.md", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/assert.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/assert/strict.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/async_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/buffer.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/child_process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/cluster.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/console.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/constants.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/crypto.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dgram.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/diagnostics_channel.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dns.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dns/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dom-events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/domain.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/fs.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/fs/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/globals.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/globals.global.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/http.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/http2.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/https.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/index.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/inspector.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/module.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/net.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/os.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/package.json", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/path.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/perf_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/punycode.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/querystring.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/readline.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/readline/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/repl.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream/consumers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream/web.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/string_decoder.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/test.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/timers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/timers/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/tls.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/trace_events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/assert.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/assert/strict.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/async_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/buffer.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/child_process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/cluster.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/console.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/constants.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/crypto.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dgram.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/diagnostics_channel.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dns.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dns/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dom-events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/domain.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/fs.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/fs/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/globals.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/globals.global.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/http.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/http2.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/https.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/index.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/inspector.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/module.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/net.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/os.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/path.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/perf_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/punycode.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/querystring.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/readline.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/readline/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/repl.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream/consumers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream/web.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/string_decoder.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/test.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/timers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/timers/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/tls.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/trace_events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/tty.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/url.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/util.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/v8.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/vm.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/wasi.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/worker_threads.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/zlib.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/tty.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/url.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/util.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/v8.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/vm.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/wasi.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/worker_threads.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/zlib.d.ts", + "create-links.js", + ], + "links": Array [ + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-01/node_modules/@types/node", + "targetPath": "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-01/node_modules/package-extractor-test-02", + "targetPath": "build-tests/package-extractor-test-02", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-01/node_modules/package-extractor-test-03", + "targetPath": "build-tests/package-extractor-test-03", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-02/node_modules/package-extractor-test-03", + "targetPath": "build-tests/package-extractor-test-03", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-03/node_modules/@types/node", + "targetPath": "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node", + }, + ], + "mainProjectName": "package-extractor-test-01", + "projects": Array [ + Object { + "path": "build-tests/package-extractor-test-01", + "projectName": "package-extractor-test-01", + }, + Object { + "path": "build-tests/package-extractor-test-02", + "projectName": "package-extractor-test-02", + }, + Object { + "path": "build-tests/package-extractor-test-03", + "projectName": "package-extractor-test-03", + }, + ], +} +`; + +exports[`PackageExtractor should extract project with script linkCreation and custom linkCreationScriptPath 1`] = ` +Object { + "files": Array [ + "build-tests/package-extractor-test-01/.rush/temp/shrinkwrap-deps.json", + "build-tests/package-extractor-test-01/package.json", + "build-tests/package-extractor-test-01/src/index.js", + "build-tests/package-extractor-test-01/src/subdir/file.js", + "build-tests/package-extractor-test-02/.rush/temp/shrinkwrap-deps.json", + "build-tests/package-extractor-test-02/package.json", + "build-tests/package-extractor-test-02/src/index.js", + "build-tests/package-extractor-test-03/.rush/temp/shrinkwrap-deps.json", + "build-tests/package-extractor-test-03/package.json", + "build-tests/package-extractor-test-03/src/index.js", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/LICENSE", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/README.md", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/assert.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/assert/strict.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/async_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/buffer.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/child_process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/cluster.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/console.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/constants.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/crypto.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/dgram.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/diagnostics_channel.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/dns.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/dns/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/domain.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/fs.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/fs/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/globals.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/globals.global.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/http.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/http2.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/https.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/index.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/inspector.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/module.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/net.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/os.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/package.json", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/path.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/perf_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/punycode.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/querystring.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/readline.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/repl.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream/consumers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/stream/web.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/string_decoder.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/timers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/timers/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/tls.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/trace_events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/tty.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/url.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/util.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/v8.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/vm.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/wasi.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/worker_threads.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node/zlib.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/LICENSE", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/README.md", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/assert.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/assert/strict.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/async_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/buffer.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/child_process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/cluster.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/console.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/constants.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/crypto.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dgram.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/diagnostics_channel.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dns.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dns/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/dom-events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/domain.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/fs.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/fs/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/globals.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/globals.global.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/http.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/http2.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/https.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/index.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/inspector.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/module.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/net.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/os.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/package.json", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/path.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/perf_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/punycode.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/querystring.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/readline.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/readline/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/repl.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream/consumers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/stream/web.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/string_decoder.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/test.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/timers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/timers/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/tls.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/trace_events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/assert.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/assert/strict.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/async_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/buffer.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/child_process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/cluster.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/console.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/constants.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/crypto.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dgram.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/diagnostics_channel.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dns.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dns/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/dom-events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/domain.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/fs.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/fs/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/globals.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/globals.global.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/http.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/http2.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/https.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/index.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/inspector.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/module.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/net.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/os.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/path.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/perf_hooks.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/process.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/punycode.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/querystring.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/readline.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/readline/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/repl.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream/consumers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/stream/web.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/string_decoder.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/test.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/timers.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/timers/promises.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/tls.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/trace_events.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/tty.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/url.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/util.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/v8.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/vm.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/wasi.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/worker_threads.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/ts4.8/zlib.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/tty.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/url.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/util.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/v8.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/vm.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/wasi.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/worker_threads.d.ts", + "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node/zlib.d.ts", + "foo/bar/baz.js", + ], + "links": Array [ + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-01/node_modules/@types/node", + "targetPath": "common/temp/default/node_modules/.pnpm/@types+node@18.17.15/node_modules/@types/node", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-01/node_modules/package-extractor-test-02", + "targetPath": "build-tests/package-extractor-test-02", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-01/node_modules/package-extractor-test-03", + "targetPath": "build-tests/package-extractor-test-03", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-02/node_modules/package-extractor-test-03", + "targetPath": "build-tests/package-extractor-test-03", + }, + Object { + "kind": "folderLink", + "linkPath": "build-tests/package-extractor-test-03/node_modules/@types/node", + "targetPath": "common/temp/default/node_modules/.pnpm/@types+node@17.0.41/node_modules/@types/node", + }, + ], + "mainProjectName": "package-extractor-test-01", + "projects": Array [ + Object { + "path": "build-tests/package-extractor-test-01", + "projectName": "package-extractor-test-01", + }, + Object { + "path": "build-tests/package-extractor-test-02", + "projectName": "package-extractor-test-02", + }, + Object { + "path": "build-tests/package-extractor-test-03", + "projectName": "package-extractor-test-03", + }, + ], +} +`; From e64a2696f695eedef7a04056f4b1fccdc1f520d2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 14 Sep 2024 08:23:43 +0000 Subject: [PATCH 2/8] Cleanup --- .../package-extractor/src/AssetHandler.ts | 32 ++++--- .../package-extractor/src/PackageExtractor.ts | 85 +++++++------------ libraries/package-extractor/src/index.ts | 1 + 3 files changed, 50 insertions(+), 68 deletions(-) diff --git a/libraries/package-extractor/src/AssetHandler.ts b/libraries/package-extractor/src/AssetHandler.ts index c1374a839cb..63e1d702dc4 100644 --- a/libraries/package-extractor/src/AssetHandler.ts +++ b/libraries/package-extractor/src/AssetHandler.ts @@ -6,13 +6,7 @@ import fs from 'node:fs'; import { FileSystem, Path, type FileSystemStats } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import { ArchiveManager } from './ArchiveManager'; - -export interface IAssetHandlerOptions { - terminal: ITerminal; - targetRootFolder: string; - createArchiveOnly?: boolean; - archiveFilePath?: string; -} +import type { IExtractorOptions, LinkCreationMode } from './PackageExtractor'; export interface IIncludeAssetOptions { sourceFilePath?: string; @@ -44,18 +38,32 @@ export class AssetHandler { private readonly _createArchiveOnly: boolean; private readonly _archiveManager: ArchiveManager | undefined; private readonly _archiveFilePath: string | undefined; + private readonly _linkCreationMode: LinkCreationMode; private readonly _includedAssetPaths: Set = new Set(); private _isFinalized: boolean = false; - public constructor(options: IAssetHandlerOptions) { - const { terminal, targetRootFolder, archiveFilePath, createArchiveOnly = false } = options; + public constructor(options: IExtractorOptions) { + const { + terminal, + targetRootFolder, + linkCreation, + createArchiveFilePath, + createArchiveOnly = false + } = options; this._terminal = terminal; this._targetRootFolder = targetRootFolder; - this._archiveFilePath = archiveFilePath; - this._createArchiveOnly = createArchiveOnly; - if (this._archiveFilePath) { + if (createArchiveFilePath) { + if (path.extname(createArchiveFilePath) !== '.zip') { + throw new Error('Only archives with the .zip file extension are currently supported.'); + } + this._archiveFilePath = path.resolve(targetRootFolder, createArchiveFilePath); this._archiveManager = new ArchiveManager(); } + if (createArchiveOnly && !this._archiveManager) { + throw new Error('createArchiveOnly cannot be true if createArchiveFilePath is not provided'); + } + this._createArchiveOnly = createArchiveOnly; + this._linkCreationMode = linkCreation || 'default'; } public async includeAssetAsync(options: IIncludeAssetPathOptions): Promise; diff --git a/libraries/package-extractor/src/PackageExtractor.ts b/libraries/package-extractor/src/PackageExtractor.ts index 07c89b0eb44..f0f1a47c624 100644 --- a/libraries/package-extractor/src/PackageExtractor.ts +++ b/libraries/package-extractor/src/PackageExtractor.ts @@ -177,6 +177,13 @@ export interface IExtractorDependencyConfiguration { patternsToInclude?: string[]; } +/** + * The mode to use for link creation. + * + * @public + */ +export type LinkCreationMode = 'default' | 'script' | 'none'; + /** * Options that can be provided to the extractor. * @@ -259,7 +266,7 @@ export interface IExtractorOptions { * to create links on the server machine, after the files have been uploaded. * "none": Do nothing; some other tool may create the links later, based on the extractor-metadata.json file. */ - linkCreation?: 'default' | 'script' | 'none'; + linkCreation?: LinkCreationMode; /** * The path to the generated link creation script. This is only used when {@link IExtractorOptions.linkCreation} @@ -331,52 +338,22 @@ export class PackageExtractor { targetRootFolder, mainProjectName, overwriteExisting, - createArchiveFilePath, - createArchiveOnly, dependencyConfigurations } = options; - if (createArchiveOnly) { - if (options.linkCreation !== 'script' && options.linkCreation !== 'none') { - throw new Error('createArchiveOnly is only supported when linkCreation is "script" or "none"'); - } - if (!createArchiveFilePath) { - throw new Error('createArchiveOnly is only supported when createArchiveFilePath is specified'); - } - } - - let archiveFilePath: string | undefined; - if (createArchiveFilePath) { - if (path.extname(createArchiveFilePath) !== '.zip') { - throw new Error('Only archives with the .zip file extension are currently supported.'); - } - - archiveFilePath = path.resolve(targetRootFolder, createArchiveFilePath); - } - - await FileSystem.ensureFolderAsync(targetRootFolder); - terminal.writeLine(Colorize.cyan(`Extracting to target folder: ${targetRootFolder}`)); terminal.writeLine(Colorize.cyan(`Main project for extraction: ${mainProjectName}`)); - try { - const existingExtraction: boolean = - (await FileSystem.readFolderItemNamesAsync(targetRootFolder)).length > 0; - if (existingExtraction) { - if (!overwriteExisting) { - throw new Error( - 'The extraction target folder is not empty. Overwrite must be explicitly requested' - ); - } else { - terminal.writeLine('Deleting target folder contents...'); - terminal.writeLine(''); - await FileSystem.ensureEmptyFolderAsync(targetRootFolder); - } - } - } catch (error: unknown) { - if (!FileSystem.isFolderDoesNotExistError(error as Error)) { - throw error; + await FileSystem.ensureFolderAsync(targetRootFolder); + const existingExtraction: boolean = + (await FileSystem.readFolderItemNamesAsync(targetRootFolder)).length > 0; + if (existingExtraction) { + if (!overwriteExisting) { + throw new Error('The extraction target folder is not empty. Overwrite must be explicitly requested'); } + terminal.writeLine('Deleting target folder contents...'); + terminal.writeLine(''); + await FileSystem.ensureEmptyFolderAsync(targetRootFolder); } // Create a new state for each run @@ -387,12 +364,7 @@ export class PackageExtractor { projectConfigurationsByPath: new Map(projectConfigurations.map((p) => [p.projectFolder, p])), dependencyConfigurationsByName: new Map(), symlinkAnalyzer: new SymlinkAnalyzer({ requiredSourceParentPath: sourceRootFolder }), - assetHandler: new AssetHandler({ - terminal, - targetRootFolder, - createArchiveOnly, - archiveFilePath - }) + assetHandler: new AssetHandler(options) }; // set state dependencyConfigurationsByName @@ -508,6 +480,15 @@ export class PackageExtractor { } switch (linkCreation) { + case 'default': { + terminal.writeLine('Creating symlinks'); + const linksToCopy: ILinkInfo[] = symlinkAnalyzer.reportSymlinks(); + await Async.forEachAsync(linksToCopy, async (linkToCopy: ILinkInfo) => { + await this._extractSymlinkAsync(linkToCopy, options, state); + }); + await this._makeBinLinksAsync(options, state); + break; + } case 'script': { terminal.writeLine(`Creating ${createLinksScriptFilename}`); const createLinksSourceFilePath: string = `${scriptsFolderPath}/${createLinksScriptFilename}`; @@ -526,15 +507,7 @@ export class PackageExtractor { }); break; } - case 'default': { - terminal.writeLine('Creating symlinks'); - const linksToCopy: ILinkInfo[] = symlinkAnalyzer.reportSymlinks(); - await Async.forEachAsync(linksToCopy, async (linkToCopy: ILinkInfo) => { - await this._extractSymlinkAsync(linkToCopy, options, state); - }); - await this._makeBinLinksAsync(options, state); - break; - } + case 'none': default: { break; } @@ -889,7 +862,7 @@ export class PackageExtractor { const { kind, linkStats: sourceFileStats } = await state.symlinkAnalyzer.analyzePathAsync({ inputPath: sourceFilePath }); - if (kind !== 'link') { + if (kind === 'file') { const targetFilePath: string = path.resolve(targetFolderPath, npmPackFile); await assetHandler.includeAssetAsync({ sourceFilePath, diff --git a/libraries/package-extractor/src/index.ts b/libraries/package-extractor/src/index.ts index f4e882d9be1..564800a2f3c 100644 --- a/libraries/package-extractor/src/index.ts +++ b/libraries/package-extractor/src/index.ts @@ -3,6 +3,7 @@ export { PackageExtractor, + type LinkCreationMode, type IExtractorOptions, type IExtractorProjectConfiguration, type IExtractorDependencyConfiguration, From d759532430746d8ed422d49885ecbe7c3b13512a Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 14 Sep 2024 08:24:30 +0000 Subject: [PATCH 3/8] Update API --- common/reviews/api/package-extractor.api.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/reviews/api/package-extractor.api.md b/common/reviews/api/package-extractor.api.md index b756b381184..f0503a30d4f 100644 --- a/common/reviews/api/package-extractor.api.md +++ b/common/reviews/api/package-extractor.api.md @@ -31,7 +31,7 @@ export interface IExtractorOptions { folderToCopy?: string; includeDevDependencies?: boolean; includeNpmIgnoreFiles?: boolean; - linkCreation?: 'default' | 'script' | 'none'; + linkCreation?: LinkCreationMode; linkCreationScriptPath?: string; mainProjectName: string; overwriteExisting: boolean; @@ -75,6 +75,9 @@ export interface IProjectInfoJson { projectName: string; } +// @public +export type LinkCreationMode = 'default' | 'script' | 'none'; + // @public export class PackageExtractor { extractAsync(options: IExtractorOptions): Promise; From 88775a71c3a1acf1cf3c6669362a8bf400a74e48 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 14 Sep 2024 08:28:58 +0000 Subject: [PATCH 4/8] Rush change --- .../user-danade-AddFileTracking_2024-09-14-08-27.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json diff --git a/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json b/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json new file mode 100644 index 00000000000..3e7d6ad2fc5 --- /dev/null +++ b/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/package-extractor", + "comment": "Add a \"files\" field to the extractor-metadata.json file", + "type": "minor" + } + ], + "packageName": "@rushstack/package-extractor" +} \ No newline at end of file From 16bae91daba0fe74ab6fdef886bd278c43e60fc6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 14 Sep 2024 09:25:14 +0000 Subject: [PATCH 5/8] More cleanup --- .../package-extractor/src/AssetHandler.ts | 79 ++++++- .../package-extractor/src/PackageExtractor.ts | 204 +++++------------- libraries/package-extractor/src/Utils.ts | 40 +++- 3 files changed, 166 insertions(+), 157 deletions(-) diff --git a/libraries/package-extractor/src/AssetHandler.ts b/libraries/package-extractor/src/AssetHandler.ts index 63e1d702dc4..d565bb9a3f0 100644 --- a/libraries/package-extractor/src/AssetHandler.ts +++ b/libraries/package-extractor/src/AssetHandler.ts @@ -3,10 +3,12 @@ import path from 'node:path'; import fs from 'node:fs'; -import { FileSystem, Path, type FileSystemStats } from '@rushstack/node-core-library'; +import { Async, FileSystem, Path, type FileSystemStats } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import { ArchiveManager } from './ArchiveManager'; import type { IExtractorOptions, LinkCreationMode } from './PackageExtractor'; +import type { ILinkInfo, SymlinkAnalyzer } from './SymlinkAnalyzer'; +import { remapSourcePathForTargetFolder } from './Utils'; export interface IIncludeAssetOptions { sourceFilePath?: string; @@ -32,26 +34,36 @@ export interface IIncludeAssetContentOptions extends IIncludeAssetOptions { sourceFileStats?: never; } +export interface IAssetHandlerOptions extends IExtractorOptions { + symlinkAnalyzer: SymlinkAnalyzer; +} + export class AssetHandler { private readonly _terminal: ITerminal; + private readonly _sourceRootFolder: string; private readonly _targetRootFolder: string; private readonly _createArchiveOnly: boolean; + private readonly _symlinkAnalyzer: SymlinkAnalyzer; private readonly _archiveManager: ArchiveManager | undefined; private readonly _archiveFilePath: string | undefined; private readonly _linkCreationMode: LinkCreationMode; private readonly _includedAssetPaths: Set = new Set(); private _isFinalized: boolean = false; - public constructor(options: IExtractorOptions) { + public constructor(options: IAssetHandlerOptions) { const { terminal, + sourceRootFolder, targetRootFolder, linkCreation, + symlinkAnalyzer, createArchiveFilePath, createArchiveOnly = false } = options; this._terminal = terminal; + this._sourceRootFolder = sourceRootFolder; this._targetRootFolder = targetRootFolder; + this._symlinkAnalyzer = symlinkAnalyzer; if (createArchiveFilePath) { if (path.extname(createArchiveFilePath) !== '.zip') { throw new Error('Only archives with the .zip file extension are currently supported.'); @@ -140,10 +152,73 @@ export class AssetHandler { if (this._isFinalized) { throw new Error('finalizeAsync() has already been called'); } + + if (this._linkCreationMode === 'default') { + this._terminal.writeLine('Creating symlinks'); + const linksToCopy: ILinkInfo[] = this._symlinkAnalyzer.reportSymlinks(); + await Async.forEachAsync(linksToCopy, async (linkToCopy: ILinkInfo) => { + await this._extractSymlinkAsync(linkToCopy); + }); + } + if (this._archiveManager && this._archiveFilePath) { this._terminal.writeLine(`Creating archive at "${this._archiveFilePath}"`); await this._archiveManager.createArchiveAsync(this._archiveFilePath); } + this._isFinalized = true; } + + /** + * Create a symlink as described by the ILinkInfo object. + */ + private async _extractSymlinkAsync(linkInfo: ILinkInfo): Promise { + const { kind, linkPath, targetPath } = { + ...linkInfo, + linkPath: remapSourcePathForTargetFolder({ + sourceRootFolder: this._sourceRootFolder, + targetRootFolder: this._targetRootFolder, + sourcePath: linkInfo.linkPath + }), + targetPath: remapSourcePathForTargetFolder({ + sourceRootFolder: this._sourceRootFolder, + targetRootFolder: this._targetRootFolder, + sourcePath: linkInfo.targetPath + }) + }; + + const newLinkFolder: string = path.dirname(linkPath); + await FileSystem.ensureFolderAsync(newLinkFolder); + + // Link to the relative path for symlinks + const relativeTargetPath: string = path.relative(newLinkFolder, targetPath); + + // NOTE: This logic is based on NpmLinkManager._createSymlink() + if (kind === 'fileLink') { + // For files, we use a Windows "hard link", because creating a symbolic link requires + // administrator permission. However hard links seem to cause build failures on Mac, + // so for all other operating systems we use symbolic links for this case. + if (process.platform === 'win32') { + await FileSystem.createHardLinkAsync({ + linkTargetPath: relativeTargetPath, + newLinkPath: linkPath + }); + } else { + await FileSystem.createSymbolicLinkFileAsync({ + linkTargetPath: relativeTargetPath, + newLinkPath: linkPath + }); + } + } else { + // Junctions are only supported on Windows. This will create a symbolic link on other platforms. + await FileSystem.createSymbolicLinkJunctionAsync({ + linkTargetPath: relativeTargetPath, + newLinkPath: linkPath + }); + } + + // Since the created symlinks have the required relative paths, they can be added directly to + // the archive. + await this.includeAssetAsync({ targetFilePath: linkPath }); + } } diff --git a/libraries/package-extractor/src/PackageExtractor.ts b/libraries/package-extractor/src/PackageExtractor.ts index f0f1a47c624..1c8b4615a07 100644 --- a/libraries/package-extractor/src/PackageExtractor.ts +++ b/libraries/package-extractor/src/PackageExtractor.ts @@ -20,7 +20,7 @@ import { Colorize, type ITerminal } from '@rushstack/terminal'; import { SymlinkAnalyzer, type ILinkInfo, type PathNode } from './SymlinkAnalyzer'; import { AssetHandler } from './AssetHandler'; -import { matchesWithStar } from './Utils'; +import { matchesWithStar, remapSourcePathForTargetFolder, remapPathForExtractorMetadata } from './Utils'; import { createLinksScriptFilename, scriptsFolderPath } from './PathConstants'; // (@types/npm-packlist is missing this API) @@ -357,14 +357,17 @@ export class PackageExtractor { } // Create a new state for each run + const symlinkAnalyzer: SymlinkAnalyzer = new SymlinkAnalyzer({ + requiredSourceParentPath: sourceRootFolder + }); const state: IExtractorState = { + symlinkAnalyzer, + assetHandler: new AssetHandler({ ...options, symlinkAnalyzer }), foldersToCopy: new Set(), packageJsonByPath: new Map(), projectConfigurationsByName: new Map(projectConfigurations.map((p) => [p.projectName, p])), projectConfigurationsByPath: new Map(projectConfigurations.map((p) => [p.projectFolder, p])), - dependencyConfigurationsByName: new Map(), - symlinkAnalyzer: new SymlinkAnalyzer({ requiredSourceParentPath: sourceRootFolder }), - assetHandler: new AssetHandler(options) + dependencyConfigurationsByName: new Map() }; // set state dependencyConfigurationsByName @@ -421,10 +424,9 @@ export class PackageExtractor { targetRootFolder, folderToCopy: additionalFolderToCopy, linkCreation, - linkCreationScriptPath, createArchiveOnly } = options; - const { projectConfigurationsByName, foldersToCopy, assetHandler, symlinkAnalyzer } = state; + const { projectConfigurationsByName, foldersToCopy } = state; const mainProjectConfiguration: IExtractorProjectConfiguration | undefined = projectConfigurationsByName.get(mainProjectName); @@ -479,38 +481,10 @@ export class PackageExtractor { await this._extractFolderAsync(additionalFolderPath, additionalFolderExtractorOptions, state); } - switch (linkCreation) { - case 'default': { - terminal.writeLine('Creating symlinks'); - const linksToCopy: ILinkInfo[] = symlinkAnalyzer.reportSymlinks(); - await Async.forEachAsync(linksToCopy, async (linkToCopy: ILinkInfo) => { - await this._extractSymlinkAsync(linkToCopy, options, state); - }); - await this._makeBinLinksAsync(options, state); - break; - } - case 'script': { - terminal.writeLine(`Creating ${createLinksScriptFilename}`); - const createLinksSourceFilePath: string = `${scriptsFolderPath}/${createLinksScriptFilename}`; - const createLinksTargetFilePath: string = path.resolve( - targetRootFolder, - linkCreationScriptPath || createLinksScriptFilename - ); - let createLinksScriptContent: string = await FileSystem.readFileAsync(createLinksSourceFilePath); - createLinksScriptContent = createLinksScriptContent.replace( - TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, - Path.convertToSlashes(path.relative(path.dirname(createLinksTargetFilePath), targetRootFolder)) - ); - await assetHandler.includeAssetAsync({ - sourceFileContent: createLinksScriptContent, - targetFilePath: createLinksTargetFilePath - }); - break; - } - case 'none': - default: { - break; - } + if (linkCreation === 'default') { + await this._makeBinLinksAsync(options, state); + } else if (linkCreation === 'script') { + await this._writeCreateLinksScriptAsync(options, state); } terminal.writeLine('Creating extractor-metadata.json'); @@ -706,61 +680,6 @@ export class PackageExtractor { return allDependencyNames; } - /** - * Maps a file path from IExtractorOptions.sourceRootFolder to IExtractorOptions.targetRootFolder - * - * Example input: "C:\\MyRepo\\libraries\\my-lib" - * Example output: "C:\\MyRepo\\common\\deploy\\libraries\\my-lib" - */ - private _remapSourcePathForExtractorFolder( - absolutePathInSourceFolder: string, - options: IExtractorOptions - ): string { - const { sourceRootFolder, targetRootFolder } = options; - const relativePath: string = path.relative(sourceRootFolder, absolutePathInSourceFolder); - if (relativePath.startsWith('..')) { - throw new Error(`Source path "${absolutePathInSourceFolder}" is not under "${sourceRootFolder}"`); - } - const absolutePathInTargetFolder: string = path.join(targetRootFolder, relativePath); - return absolutePathInTargetFolder; - } - - /** - * Maps a file path from IExtractorOptions.sourceRootFolder to relative path - * - * Example input: "C:\\MyRepo\\libraries\\my-lib" - * Example output: "libraries/my-lib" - */ - private _remapSourcePathForExtractorMetadata( - absolutePathInSourceFolder: string, - options: IExtractorOptions - ): string { - const { sourceRootFolder } = options; - return this._remapPathForExtractorMetadata(sourceRootFolder, absolutePathInSourceFolder); - } - - /** - * Maps a file path from IExtractorOptions.targetRootFolder to relative path - * - * Example input: "C:\\MyRepo\\libraries\\my-lib" - * Example output: "libraries/my-lib" - */ - private _remapTargetPathForExtractorMetadata( - absolutePathInTargetFolder: string, - options: IExtractorOptions - ): string { - const { targetRootFolder } = options; - return this._remapPathForExtractorMetadata(targetRootFolder, absolutePathInTargetFolder); - } - - private _remapPathForExtractorMetadata(folderPath: string, filePath: string): string { - const relativePath: string = path.relative(folderPath, filePath); - if (relativePath.startsWith('..')) { - throw new Error(`Path "${filePath}" is not under "${folderPath}"`); - } - return Path.convertToSlashes(relativePath); - } - /** * Copy one package folder to the extractor target folder. */ @@ -840,7 +759,10 @@ export class PackageExtractor { useNpmIgnoreFilter = true; } - const targetFolderPath: string = this._remapSourcePathForExtractorFolder(sourceFolderPath, options); + const targetFolderPath: string = remapSourcePathForTargetFolder({ + ...options, + sourcePath: sourceFolderPath + }); if (useNpmIgnoreFilter) { const npmPackFiles: string[] = await PackageExtractor.getPackageIncludedFilesAsync(sourceFolderPath); await Async.forEachAsync( @@ -946,57 +868,6 @@ export class PackageExtractor { } } - /** - * Create a symlink as described by the ILinkInfo object. - */ - private async _extractSymlinkAsync( - originalLinkInfo: ILinkInfo, - options: IExtractorOptions, - state: IExtractorState - ): Promise { - const { kind, linkPath, targetPath } = { - kind: originalLinkInfo.kind, - linkPath: this._remapSourcePathForExtractorFolder(originalLinkInfo.linkPath, options), - targetPath: this._remapSourcePathForExtractorFolder(originalLinkInfo.targetPath, options) - }; - - const newLinkFolder: string = path.dirname(linkPath); - await FileSystem.ensureFolderAsync(newLinkFolder); - - // Link to the relative path for symlinks - const relativeTargetPath: string = path.relative(newLinkFolder, targetPath); - - // NOTE: This logic is based on NpmLinkManager._createSymlink() - if (kind === 'fileLink') { - // For files, we use a Windows "hard link", because creating a symbolic link requires - // administrator permission. However hard links seem to cause build failures on Mac, - // so for all other operating systems we use symbolic links for this case. - if (process.platform === 'win32') { - await FileSystem.createHardLinkAsync({ - linkTargetPath: relativeTargetPath, - newLinkPath: linkPath - }); - } else { - await FileSystem.createSymbolicLinkFileAsync({ - linkTargetPath: relativeTargetPath, - newLinkPath: linkPath - }); - } - } else { - // Junctions are only supported on Windows. This will create a symbolic link on other platforms. - await FileSystem.createSymbolicLinkJunctionAsync({ - linkTargetPath: relativeTargetPath, - newLinkPath: linkPath - }); - } - - // Since the created symlinks have the required relative paths, they can be added directly to - // the archive. - await state.assetHandler.includeAssetAsync({ - targetFilePath: linkPath - }); - } - /** * Write the common/deploy/deploy-metadata.json file. */ @@ -1004,7 +875,8 @@ export class PackageExtractor { options: IExtractorOptions, state: IExtractorState ): Promise { - const { mainProjectName, targetRootFolder, linkCreation, linkCreationScriptPath } = options; + const { mainProjectName, sourceRootFolder, targetRootFolder, linkCreation, linkCreationScriptPath } = + options; const { projectConfigurationsByPath } = state; const extractorMetadataFileName: string = 'extractor-metadata.json'; @@ -1027,7 +899,7 @@ export class PackageExtractor { if (state.foldersToCopy.has(projectFolder)) { extractorMetadataJson.projects.push({ projectName, - path: this._remapSourcePathForExtractorMetadata(projectFolder, options) + path: remapPathForExtractorMetadata(sourceRootFolder, projectFolder) }); } } @@ -1036,13 +908,13 @@ export class PackageExtractor { for (const { kind, linkPath, targetPath } of state.symlinkAnalyzer.reportSymlinks()) { extractorMetadataJson.links.push({ kind, - linkPath: this._remapSourcePathForExtractorMetadata(linkPath, options), - targetPath: this._remapSourcePathForExtractorMetadata(targetPath, options) + linkPath: remapPathForExtractorMetadata(sourceRootFolder, linkPath), + targetPath: remapPathForExtractorMetadata(sourceRootFolder, targetPath) }); } for (const assetPath of state.assetHandler.assetPaths) { - extractorMetadataJson.files.push(this._remapTargetPathForExtractorMetadata(assetPath, options)); + extractorMetadataJson.files.push(remapPathForExtractorMetadata(targetRootFolder, assetPath)); } const extractorMetadataFileContent: string = JSON.stringify(extractorMetadataJson, undefined, 0); @@ -1062,10 +934,10 @@ export class PackageExtractor { await Async.forEachAsync( extractedProjectFolders, async (projectFolder: string) => { - const extractedProjectFolder: string = this._remapSourcePathForExtractorFolder( - projectFolder, - options - ); + const extractedProjectFolder: string = remapSourcePathForTargetFolder({ + ...options, + sourcePath: projectFolder + }); const extractedProjectNodeModulesFolder: string = path.join(extractedProjectFolder, 'node_modules'); const extractedProjectBinFolder: string = path.join(extractedProjectNodeModulesFolder, '.bin'); @@ -1093,4 +965,28 @@ export class PackageExtractor { } ); } + + private async _writeCreateLinksScriptAsync( + options: IExtractorOptions, + state: IExtractorState + ): Promise { + const { terminal, targetRootFolder, linkCreationScriptPath } = options; + const { assetHandler } = state; + + terminal.writeLine(`Creating ${createLinksScriptFilename}`); + const createLinksSourceFilePath: string = `${scriptsFolderPath}/${createLinksScriptFilename}`; + const createLinksTargetFilePath: string = path.resolve( + targetRootFolder, + linkCreationScriptPath || createLinksScriptFilename + ); + let createLinksScriptContent: string = await FileSystem.readFileAsync(createLinksSourceFilePath); + createLinksScriptContent = createLinksScriptContent.replace( + TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, + Path.convertToSlashes(path.relative(path.dirname(createLinksTargetFilePath), targetRootFolder)) + ); + await assetHandler.includeAssetAsync({ + sourceFileContent: createLinksScriptContent, + targetFilePath: createLinksTargetFilePath + }); + } } diff --git a/libraries/package-extractor/src/Utils.ts b/libraries/package-extractor/src/Utils.ts index 6ec4d98cbd0..ed3eeae2df7 100644 --- a/libraries/package-extractor/src/Utils.ts +++ b/libraries/package-extractor/src/Utils.ts @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { Text } from '@rushstack/node-core-library'; +import path from 'node:path'; +import { Path, Text } from '@rushstack/node-core-library'; export function matchesWithStar(patternWithStar: string, input: string): boolean { // Map "@types/*" --> "^\@types\/.*$" @@ -16,3 +17,40 @@ export function matchesWithStar(patternWithStar: string, input: string): boolean const regExp: RegExp = new RegExp(pattern); return regExp.test(input); } + +export interface IRemapPathForTargetFolder { + sourcePath: string; + sourceRootFolder: string; + targetRootFolder: string; +} + +/** + * Maps a file path under the provided {@link IRemapPathForTargetFolder.sourceRootFolder} to the provided + * {@link IExtractorOptions.targetRootFolder}. + * + * Example input: "C:\\MyRepo\\libraries\\my-lib" + * Example output: "C:\\MyRepo\\common\\deploy\\libraries\\my-lib" + */ +export function remapSourcePathForTargetFolder(options: IRemapPathForTargetFolder): string { + const { sourcePath, sourceRootFolder, targetRootFolder } = options; + const relativePath: string = path.relative(sourceRootFolder, sourcePath); + if (relativePath.startsWith('..')) { + throw new Error(`Source path "${sourcePath}" is not under "${sourceRootFolder}"`); + } + const absolutePathInTargetFolder: string = path.join(targetRootFolder, relativePath); + return absolutePathInTargetFolder; +} + +/** + * Maps a file path under the provided folder path to the expected path format for the extractor metadata. + * + * Example input: "C:\\MyRepo\\libraries\\my-lib" + * Example output: "common/deploy/libraries/my-lib" + */ +export function remapPathForExtractorMetadata(folderPath: string, filePath: string): string { + const relativePath: string = path.relative(folderPath, filePath); + if (relativePath.startsWith('..')) { + throw new Error(`Path "${filePath}" is not under "${folderPath}"`); + } + return Path.convertToSlashes(relativePath); +} From 6af6d0536f7e12f13e0aa371d3b1745b370118b3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 14 Sep 2024 18:10:12 +0000 Subject: [PATCH 6/8] Add file realization to create-links.js and make the file run async --- .../src/scripts/create-links.ts | 192 +++++++++--------- 1 file changed, 98 insertions(+), 94 deletions(-) diff --git a/libraries/package-extractor/src/scripts/create-links.ts b/libraries/package-extractor/src/scripts/create-links.ts index fde99a9aa7d..874838b5ba8 100644 --- a/libraries/package-extractor/src/scripts/create-links.ts +++ b/libraries/package-extractor/src/scripts/create-links.ts @@ -3,112 +3,111 @@ // THIS SCRIPT IS GENERATED BY THE "rush deploy" COMMAND. -/* eslint-disable no-console */ - -import * as fs from 'fs'; -import * as path from 'path'; -import type { IFileSystemCreateLinkOptions } from '@rushstack/node-core-library'; +import fs from 'node:fs'; +import path from 'node:path'; +import { Async, FileSystem, Path } from '@rushstack/node-core-library'; +import { Terminal, ConsoleTerminalProvider } from '@rushstack/terminal'; import type { TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING as TargetRootScriptRelativePathTemplateString, IExtractorMetadataJson } from '../PackageExtractor'; +const CREATE_ACTION_NAME: 'create' = 'create'; +const REMOVE_ACTION_NAME: 'remove' = 'remove'; +const REALIZE_FILES_OPTION_NAME: '--realize-files' = '--realize-files'; + const TARGET_ROOT_SCRIPT_RELATIVE_PATH: typeof TargetRootScriptRelativePathTemplateString = '{TARGET_ROOT_SCRIPT_RELATIVE_PATH}'; const TARGET_ROOT_FOLDER: string = path.resolve(__dirname, TARGET_ROOT_SCRIPT_RELATIVE_PATH); -// API borrowed from @rushstack/node-core-library, since this script avoids using any -// NPM dependencies. -class FileSystem { - public static createSymbolicLinkJunction(options: IFileSystemCreateLinkOptions): void { - fs.symlinkSync(options.linkTargetPath, options.newLinkPath, 'junction'); - } - - public static createSymbolicLinkFile(options: IFileSystemCreateLinkOptions): void { - fs.symlinkSync(options.linkTargetPath, options.newLinkPath, 'file'); - } - - public static createSymbolicLinkFolder(options: IFileSystemCreateLinkOptions): void { - fs.symlinkSync(options.linkTargetPath, options.newLinkPath, 'dir'); - } - - public static createHardLink(options: IFileSystemCreateLinkOptions): void { - fs.linkSync(options.linkTargetPath, options.newLinkPath); - } +async function removeLinksAsync( + terminal: Terminal, + targetRootFolder: string, + extractorMetadataObject: IExtractorMetadataJson +): Promise { + await Async.forEachAsync( + extractorMetadataObject.links, + async ({ linkPath }) => { + const newLinkPath: string = `${targetRootFolder}/${linkPath}`; + terminal.writeVerboseLine(`Removing link at path "${newLinkPath}"`); + await FileSystem.deleteFileAsync(newLinkPath, { throwIfNotExists: false }); + }, + { concurrency: 10 } + ); } -function ensureFolder(folderPath: string): void { - if (!folderPath) { - return; - } - if (fs.existsSync(folderPath)) { - return; - } - const parentPath: string = path.dirname(folderPath); - if (parentPath && parentPath !== folderPath) { - ensureFolder(parentPath); - } - fs.mkdirSync(folderPath); -} - -function removeLinks(targetRootFolder: string, extractorMetadataObject: IExtractorMetadataJson): void { - for (const linkInfo of extractorMetadataObject.links) { +async function createLinksAsync( + terminal: Terminal, + targetRootFolder: string, + extractorMetadataObject: IExtractorMetadataJson +): Promise { + await Async.forEachAsync(extractorMetadataObject.links, async (linkInfo) => { // Link to the relative path for symlinks - const newLinkPath: string = path.join(targetRootFolder, linkInfo.linkPath); - if (fs.existsSync(newLinkPath)) { - fs.unlinkSync(newLinkPath); - } - } -} - -function createLinks(targetRootFolder: string, extractorMetadataObject: IExtractorMetadataJson): void { - for (const linkInfo of extractorMetadataObject.links) { - // Link to the relative path for symlinks - const newLinkPath: string = path.join(targetRootFolder, linkInfo.linkPath); - const linkTargetPath: string = path.join(targetRootFolder, linkInfo.targetPath); + const newLinkPath: string = `${targetRootFolder}/${linkInfo.linkPath}`; + const linkTargetPath: string = `${targetRootFolder}/${linkInfo.targetPath}`; // Make sure the containing folder exists - ensureFolder(path.dirname(newLinkPath)); + await FileSystem.ensureFolderAsync(path.dirname(newLinkPath)); // NOTE: This logic is based on NpmLinkManager._createSymlink() - if (process.platform === 'win32') { - if (linkInfo.kind === 'folderLink') { - // For directories, we use a Windows "junction". On Unix, this produces a regular symlink. - FileSystem.createSymbolicLinkJunction({ newLinkPath, linkTargetPath }); + if (linkInfo.kind === 'folderLink') { + terminal.writeVerboseLine(`Creating linked folder at path "${newLinkPath}"`); + await FileSystem.createSymbolicLinkJunctionAsync({ newLinkPath, linkTargetPath }); + } else if (linkInfo.kind === 'fileLink') { + // Use hardlinks for Windows and symlinks for other platforms since creating a symbolic link + // requires administrator permission on Windows. This may cause unexpected behaviour for consumers + // of the hardlinked files. If this becomes an issue, we may need to revisit this. + terminal.writeVerboseLine(`Creating linked file at path "${newLinkPath}"`); + if (process.platform === 'win32') { + await FileSystem.createHardLinkAsync({ newLinkPath, linkTargetPath }); } else { - // For files, we use a Windows "hard link", because creating a symbolic link requires - // administrator permission. - - // NOTE: We cannot use the relative path for hard links - FileSystem.createHardLink({ newLinkPath, linkTargetPath }); + await FileSystem.createSymbolicLinkFileAsync({ newLinkPath, linkTargetPath }); } - } else { - // However hard links seem to cause build failures on Mac, so for all other operating systems - // we use symbolic links for this case. - if (linkInfo.kind === 'folderLink') { - FileSystem.createSymbolicLinkFolder({ newLinkPath, linkTargetPath }); + } + }); +} + +async function realizeFilesAsync( + terminal: Terminal, + targetRootFolder: string, + extractorMetadataObject: IExtractorMetadataJson +): Promise { + await Async.forEachAsync(extractorMetadataObject.files, async (relativeFilePath) => { + const filePath: string = `${targetRootFolder}/${relativeFilePath}`; + const realFilePath: string = await FileSystem.getRealPathAsync(filePath); + if (!Path.isEqual(realFilePath, filePath)) { + await FileSystem.deleteFileAsync(filePath); + + // Hard links seem to cause build failures on Mac, so for all other operating + // systems we copy files. + terminal.writeVerboseLine(`Realizing file at path "${filePath}"`); + if (process.platform === 'win32') { + await FileSystem.createHardLinkAsync({ newLinkPath: filePath, linkTargetPath: realFilePath }); } else { - FileSystem.createSymbolicLinkFile({ newLinkPath, linkTargetPath }); + await FileSystem.copyFileAsync({ sourcePath: realFilePath, destinationPath: filePath }); } } - } + }); } -function showUsage(): void { - console.log('Usage:'); - console.log(' node create-links.js create'); - console.log(' node create-links.js remove'); - - console.log('\nCreates or removes the symlinks for the output folder created by "rush deploy".'); - console.log('The link information is read from "extractor-metadata.json" in the same folder.'); +function showUsage(terminal: Terminal): void { + terminal.writeLine('Usage:'); + terminal.writeLine(` node create-links.js ${CREATE_ACTION_NAME} [${REALIZE_FILES_OPTION_NAME}]`); + terminal.writeLine(` node create-links.js ${REMOVE_ACTION_NAME}`); + terminal.writeLine(''); + terminal.writeLine('Creates or removes the symlinks for the output folder created by "rush deploy".'); + terminal.writeLine('The link information is read from "extractor-metadata.json" in the same folder.'); } -function main(): boolean { +async function main(terminal: Terminal): Promise { // Example: [ "node.exe", "create-links.js", ""create" ] const args: string[] = process.argv.slice(2); - - if (args.length !== 1 || (args[0] !== 'create' && args[0] !== 'remove')) { - showUsage(); + if ( + (args[0] !== CREATE_ACTION_NAME && args[0] !== REMOVE_ACTION_NAME) || + (args[0] === CREATE_ACTION_NAME && args[1] && args[1] !== REALIZE_FILES_OPTION_NAME) || + (args[0] === REMOVE_ACTION_NAME && args[1]) + ) { + showUsage(terminal); return false; } @@ -121,23 +120,28 @@ function main(): boolean { const extractorMetadataObject: IExtractorMetadataJson = JSON.parse(extractorMetadataJson); if (args[0] === 'create') { - console.log(`\nCreating links for extraction at path "${TARGET_ROOT_FOLDER}"`); - removeLinks(TARGET_ROOT_FOLDER, extractorMetadataObject); - createLinks(TARGET_ROOT_FOLDER, extractorMetadataObject); - } else { - console.log(`\nRemoving links for extraction at path "${TARGET_ROOT_FOLDER}"`); - removeLinks(TARGET_ROOT_FOLDER, extractorMetadataObject); + const realizeFiles: boolean = args[1] === '--realize-files'; + terminal.writeLine(`Creating links for extraction at path "${TARGET_ROOT_FOLDER}"`); + await removeLinksAsync(terminal, TARGET_ROOT_FOLDER, extractorMetadataObject); + await createLinksAsync(terminal, TARGET_ROOT_FOLDER, extractorMetadataObject); + if (realizeFiles) { + await realizeFilesAsync(terminal, TARGET_ROOT_FOLDER, extractorMetadataObject); + } + } else if (args[0] === 'remove') { + terminal.writeLine(`Removing links for extraction at path "${TARGET_ROOT_FOLDER}"`); + await removeLinksAsync(terminal, TARGET_ROOT_FOLDER, extractorMetadataObject); } - console.log('The operation completed successfully.'); + terminal.writeLine('The operation completed successfully.'); return true; } -try { - process.exitCode = 1; - if (main()) { - process.exitCode = 0; - } -} catch (error) { - console.log('ERROR: ' + error); -} +process.exitCode = 1; +const terminal: Terminal = new Terminal(new ConsoleTerminalProvider()); +main(terminal) + .then((result) => { + process.exitCode = result ? 0 : 1; + }) + .catch((error) => { + terminal.writeErrorLine('ERROR: ' + error); + }); From e72f5bcb77e3ed0f0fe7ab625c9054941682269d Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 15 Sep 2024 06:47:41 +0000 Subject: [PATCH 7/8] PR feedback --- .../src/scripts/create-links.ts | 92 ++++++++++--------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/libraries/package-extractor/src/scripts/create-links.ts b/libraries/package-extractor/src/scripts/create-links.ts index 874838b5ba8..5d71eb5c8b5 100644 --- a/libraries/package-extractor/src/scripts/create-links.ts +++ b/libraries/package-extractor/src/scripts/create-links.ts @@ -3,6 +3,7 @@ // THIS SCRIPT IS GENERATED BY THE "rush deploy" COMMAND. +import os from 'node:os'; import fs from 'node:fs'; import path from 'node:path'; import { Async, FileSystem, Path } from '@rushstack/node-core-library'; @@ -12,6 +13,7 @@ import type { IExtractorMetadataJson } from '../PackageExtractor'; +const MAX_CONCURRENCY: number = os.cpus().length * 2; const CREATE_ACTION_NAME: 'create' = 'create'; const REMOVE_ACTION_NAME: 'remove' = 'remove'; const REALIZE_FILES_OPTION_NAME: '--realize-files' = '--realize-files'; @@ -32,7 +34,7 @@ async function removeLinksAsync( terminal.writeVerboseLine(`Removing link at path "${newLinkPath}"`); await FileSystem.deleteFileAsync(newLinkPath, { throwIfNotExists: false }); }, - { concurrency: 10 } + { concurrency: MAX_CONCURRENCY } ); } @@ -41,30 +43,34 @@ async function createLinksAsync( targetRootFolder: string, extractorMetadataObject: IExtractorMetadataJson ): Promise { - await Async.forEachAsync(extractorMetadataObject.links, async (linkInfo) => { - // Link to the relative path for symlinks - const newLinkPath: string = `${targetRootFolder}/${linkInfo.linkPath}`; - const linkTargetPath: string = `${targetRootFolder}/${linkInfo.targetPath}`; - - // Make sure the containing folder exists - await FileSystem.ensureFolderAsync(path.dirname(newLinkPath)); - - // NOTE: This logic is based on NpmLinkManager._createSymlink() - if (linkInfo.kind === 'folderLink') { - terminal.writeVerboseLine(`Creating linked folder at path "${newLinkPath}"`); - await FileSystem.createSymbolicLinkJunctionAsync({ newLinkPath, linkTargetPath }); - } else if (linkInfo.kind === 'fileLink') { - // Use hardlinks for Windows and symlinks for other platforms since creating a symbolic link - // requires administrator permission on Windows. This may cause unexpected behaviour for consumers - // of the hardlinked files. If this becomes an issue, we may need to revisit this. - terminal.writeVerboseLine(`Creating linked file at path "${newLinkPath}"`); - if (process.platform === 'win32') { - await FileSystem.createHardLinkAsync({ newLinkPath, linkTargetPath }); - } else { - await FileSystem.createSymbolicLinkFileAsync({ newLinkPath, linkTargetPath }); + await Async.forEachAsync( + extractorMetadataObject.links, + async (linkInfo) => { + // Link to the relative path for symlinks + const newLinkPath: string = `${targetRootFolder}/${linkInfo.linkPath}`; + const linkTargetPath: string = `${targetRootFolder}/${linkInfo.targetPath}`; + + // Make sure the containing folder exists + await FileSystem.ensureFolderAsync(path.dirname(newLinkPath)); + + // NOTE: This logic is based on NpmLinkManager._createSymlink() + if (linkInfo.kind === 'folderLink') { + terminal.writeVerboseLine(`Creating linked folder at path "${newLinkPath}"`); + await FileSystem.createSymbolicLinkJunctionAsync({ newLinkPath, linkTargetPath }); + } else if (linkInfo.kind === 'fileLink') { + // Use hardlinks for Windows and symlinks for other platforms since creating a symbolic link + // requires administrator permission on Windows. This may cause unexpected behaviour for consumers + // of the hardlinked files. If this becomes an issue, we may need to revisit this. + terminal.writeVerboseLine(`Creating linked file at path "${newLinkPath}"`); + if (process.platform === 'win32') { + await FileSystem.createHardLinkAsync({ newLinkPath, linkTargetPath }); + } else { + await FileSystem.createSymbolicLinkFileAsync({ newLinkPath, linkTargetPath }); + } } - } - }); + }, + { concurrency: MAX_CONCURRENCY } + ); } async function realizeFilesAsync( @@ -72,22 +78,26 @@ async function realizeFilesAsync( targetRootFolder: string, extractorMetadataObject: IExtractorMetadataJson ): Promise { - await Async.forEachAsync(extractorMetadataObject.files, async (relativeFilePath) => { - const filePath: string = `${targetRootFolder}/${relativeFilePath}`; - const realFilePath: string = await FileSystem.getRealPathAsync(filePath); - if (!Path.isEqual(realFilePath, filePath)) { - await FileSystem.deleteFileAsync(filePath); - - // Hard links seem to cause build failures on Mac, so for all other operating - // systems we copy files. - terminal.writeVerboseLine(`Realizing file at path "${filePath}"`); - if (process.platform === 'win32') { - await FileSystem.createHardLinkAsync({ newLinkPath: filePath, linkTargetPath: realFilePath }); - } else { - await FileSystem.copyFileAsync({ sourcePath: realFilePath, destinationPath: filePath }); + await Async.forEachAsync( + extractorMetadataObject.files, + async (relativeFilePath) => { + const filePath: string = `${targetRootFolder}/${relativeFilePath}`; + const realFilePath: string = await FileSystem.getRealPathAsync(filePath); + if (!Path.isEqual(realFilePath, filePath)) { + await FileSystem.deleteFileAsync(filePath); + + // Hard links seem to cause build failures on Mac, so for all other operating + // systems we copy files. + terminal.writeVerboseLine(`Realizing file at path "${filePath}"`); + if (process.platform === 'win32') { + await FileSystem.createHardLinkAsync({ newLinkPath: filePath, linkTargetPath: realFilePath }); + } else { + await FileSystem.copyFileAsync({ sourcePath: realFilePath, destinationPath: filePath }); + } } - } - }); + }, + { concurrency: MAX_CONCURRENCY } + ); } function showUsage(terminal: Terminal): void { @@ -99,7 +109,7 @@ function showUsage(terminal: Terminal): void { terminal.writeLine('The link information is read from "extractor-metadata.json" in the same folder.'); } -async function main(terminal: Terminal): Promise { +async function runAsync(terminal: Terminal): Promise { // Example: [ "node.exe", "create-links.js", ""create" ] const args: string[] = process.argv.slice(2); if ( @@ -138,7 +148,7 @@ async function main(terminal: Terminal): Promise { process.exitCode = 1; const terminal: Terminal = new Terminal(new ConsoleTerminalProvider()); -main(terminal) +runAsync(terminal) .then((result) => { process.exitCode = result ? 0 : 1; }) From d7e054f435b102c9a85ce89f6a74eae95e740f60 Mon Sep 17 00:00:00 2001 From: Daniel <3473356+D4N14L@users.noreply.github.com> Date: Sat, 14 Sep 2024 23:48:09 -0700 Subject: [PATCH 8/8] Update common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json Co-authored-by: Ian Clanton-Thuon --- .../user-danade-AddFileTracking_2024-09-14-08-27.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json b/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json index 3e7d6ad2fc5..ea0bac67e02 100644 --- a/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json +++ b/common/changes/@rushstack/package-extractor/user-danade-AddFileTracking_2024-09-14-08-27.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/package-extractor", - "comment": "Add a \"files\" field to the extractor-metadata.json file", + "comment": "Add a `files` field to the `extractor-metadata.json` file", "type": "minor" } ],