[rush] rush-sdk try to load from global folder before install-and-run rush

common/changes/@microsoft/rush/feat-rush-sdk-load-from-global-folder_2024-06-08-08-34.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Improve `@rushtack/rush-sdk` and make it reuse `@microsoft/rush-lib` from rush global folder",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/config/rush/version-policies.json

@@ -103,7 +103,7 @@
     "policyName": "rush",
     "definitionName": "lockStepVersion",
     "version": "5.129.6",
-    "nextBump": "patch",
+    "nextBump": "minor",
     "mainProject": "@microsoft/rush"
   }
 ]

common/config/subspaces/build-tests-subspace/repo-state.json

@@ -2,5 +2,5 @@
 {
   "pnpmShrinkwrapHash": "00e9bb9c23e93f1cf44e5989a201680d8a857644",
   "preferredVersionsHash": "ce857ea0536b894ec8f346aaea08cfd85a5af648",
-  "packageJsonInjectedDependenciesHash": "3ad79132b166303fafa8efbd2fabd5055f1fe7f6"
+  "packageJsonInjectedDependenciesHash": "bc9ec80f2538e316952305db6449bf0c8cd35bf6"
 }

libraries/rush-lib/package.json

@@ -14,6 +14,13 @@
   "homepage": "https://rushjs.io",
   "main": "lib/index.js",
   "typings": "dist/rush-lib.d.ts",
+  "typesVersions": {
+    "*": {
+      "lib-esnext/*": [
+        "lib/*"
+      ]
+    }
+  },
   "scripts": {
     "build": "heft build --clean",
     "test": "heft test --clean",

libraries/rush-sdk/config/api-extractor.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
 
-  "mainEntryPointFilePath": "<projectFolder>/lib-shim/loader.d.ts",
+  "mainEntryPointFilePath": "<projectFolder>/lib-commonjs/loader.d.ts",
 
   "apiReport": {
     "enabled": true,

libraries/rush-sdk/config/heft.json

@@ -9,7 +9,7 @@
   // TODO: Add comments
   "phasesByName": {
     "build": {
-      "cleanFiles": [{ "includeGlobs": ["lib-shim"] }],
+      "cleanFiles": [{ "includeGlobs": ["lib-shim", "lib-esnext"] }],
 
       "tasksByName": {
         "copy-rush-lib-types": {
@@ -32,13 +32,20 @@
           "taskDependencies": ["copy-rush-lib-types"]
         },
 
+        "webpack": {
+          "taskDependencies": ["typescript"],
+          "taskPlugin": {
+            "pluginPackage": "@rushstack/heft-webpack5-plugin"
+          }
+        },
+
         "generate-stubs": {
           "taskDependencies": ["typescript"],
           "taskPlugin": {
             "pluginPackage": "@rushstack/heft",
             "pluginName": "run-script-plugin",
             "options": {
-              "scriptPath": "./lib-shim/generate-stubs.js"
+              "scriptPath": "./lib-commonjs/generate-stubs.js"
             }
           }
         }

libraries/rush-sdk/config/typescript.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/typescript.schema.json",
+
+  "extends": "local-node-rig/profiles/default/config/typescript.json",
+
+  "additionalModuleKindsToEmit": [
+    {
+      "moduleKind": "esnext",
+      "outFolderName": "lib-esnext"
+    }
+  ]
+}

libraries/rush-sdk/package.json

@@ -37,9 +37,12 @@
     "@microsoft/rush-lib": "workspace:*",
     "@rushstack/heft": "workspace:*",
     "local-node-rig": "workspace:*",
+    "@rushstack/heft-webpack5-plugin": "workspace:*",
     "@rushstack/stream-collator": "workspace:*",
     "@rushstack/ts-command-line": "workspace:*",
+    "@rushstack/webpack-preserve-dynamic-require-plugin": "workspace:*",
     "@types/semver": "7.5.0",
-    "@types/webpack-env": "1.18.0"
+    "@types/webpack-env": "1.18.0",
+    "webpack": "~5.82.1"
   }
 }

libraries/rush-sdk/src/index.ts

@@ -10,6 +10,7 @@ import {
   Executable
 } from '@rushstack/node-core-library';
 import { Terminal, ConsoleTerminalProvider } from '@rushstack/terminal';
+import { RushGlobalFolder } from '@microsoft/rush-lib/lib-esnext/api/RushGlobalFolder';
 import type { SpawnSyncReturns } from 'child_process';
 import {
   RUSH_LIB_NAME,
@@ -21,7 +22,9 @@ import {
   sdkContext
 } from './helpers';
 
-const verboseEnabled: boolean = typeof process !== 'undefined' && process.env.RUSH_SDK_DEBUG === '1';
+const verboseEnabled: boolean =
+  typeof process !== 'undefined' &&
+  (process.env.RUSH_SDK_DEBUG === '1' || process.env._RUSH_SDK_DEBUG === '1');
 const terminal: Terminal = new Terminal(
   new ConsoleTerminalProvider({
     verboseEnabled
@@ -31,6 +34,7 @@ const terminal: Terminal = new Terminal(
 declare const global: typeof globalThis & {
   ___rush___rushLibModule?: RushLibModuleType;
   ___rush___rushLibModuleFromEnvironment?: RushLibModuleType;
+  ___rush___rushLibModuleFromRushGlobalFolder?: RushLibModuleType;
   ___rush___rushLibModuleFromInstallAndRunRush?: RushLibModuleType;
 };
 
@@ -42,6 +46,7 @@ if (sdkContext.rushLibModule === undefined) {
   sdkContext.rushLibModule =
     global.___rush___rushLibModule ||
     global.___rush___rushLibModuleFromEnvironment ||
+    global.___rush___rushLibModuleFromRushGlobalFolder ||
     global.___rush___rushLibModuleFromInstallAndRunRush;
 }
 
@@ -111,7 +116,8 @@ if (sdkContext.rushLibModule === undefined) {
 }
 
 // SCENARIO 4:  A standalone tool or script depends on "rush-sdk", and is meant to be used inside a monorepo folder.
-// In this case, we can use install-run-rush.js to obtain the appropriate rush-lib version for the monorepo.
+// In this case, we can first load the rush-lib version in rush global folder. If the expected version is not installed,
+// using install-run-rush.js to obtain the appropriate rush-lib version for the monorepo.
 if (sdkContext.rushLibModule === undefined) {
   try {
     const rushJsonPath: string | undefined = tryFindRushJsonLocation(process.cwd());
@@ -126,51 +132,67 @@ if (sdkContext.rushLibModule === undefined) {
     const rushJson: JsonObject = JsonFile.load(rushJsonPath);
     const { rushVersion } = rushJson;
 
-    const installRunNodeModuleFolder: string = path.join(
-      monorepoRoot,
-      `common/temp/install-run/@microsoft+rush@${rushVersion}`
-    );
-
     try {
-      // First, try to load the version of "rush-lib" that was installed by install-run-rush.js
-      terminal.writeVerboseLine(`Trying to load  ${RUSH_LIB_NAME} installed by install-run-rush`);
-      sdkContext.rushLibModule = requireRushLibUnderFolderPath(installRunNodeModuleFolder);
-    } catch (e1) {
-      let installAndRunRushStderrContent: string = '';
+      terminal.writeVerboseLine(`Try to load ${RUSH_LIB_NAME} from rush global folder`);
+      const rushGlobalFolder: RushGlobalFolder = new RushGlobalFolder();
+      // The path needs to keep align with the logic inside RushVersionSelector
+      const expectedGlobalRushInstalledFolder: string = `${rushGlobalFolder.nodeSpecificPath}/rush-${rushVersion}`;
+      terminal.writeVerboseLine(
+        `The expected global rush installed folder is "${expectedGlobalRushInstalledFolder}"`
+      );
+      sdkContext.rushLibModule = requireRushLibUnderFolderPath(expectedGlobalRushInstalledFolder);
+    } catch (e) {
+      terminal.writeVerboseLine(`Failed to load ${RUSH_LIB_NAME} from rush global folder: ${e.message}`);
+    }
+
+    if (sdkContext.rushLibModule !== undefined) {
+      // to track which scenario is active and how it got initialized.
+      global.___rush___rushLibModuleFromRushGlobalFolder = sdkContext.rushLibModule;
+      terminal.writeVerboseLine(`Loaded ${RUSH_LIB_NAME} installed from rush global folder`);
+    } else {
+      const installRunNodeModuleFolder: string = `${monorepoRoot}/common/temp/install-run/@microsoft+rush@${rushVersion}`;
+
       try {
-        const installAndRunRushJSPath: string = path.join(monorepoRoot, 'common/scripts/install-run-rush.js');
+        // First, try to load the version of "rush-lib" that was installed by install-run-rush.js
+        terminal.writeVerboseLine(`Trying to load  ${RUSH_LIB_NAME} installed by install-run-rush`);
+        sdkContext.rushLibModule = requireRushLibUnderFolderPath(installRunNodeModuleFolder);
+      } catch (e1) {
+        let installAndRunRushStderrContent: string = '';
+        try {
+          const installAndRunRushJSPath: string = `${monorepoRoot}/common/scripts/install-run-rush.js`;
+
+          terminal.writeLine('The Rush engine has not been installed yet. Invoking install-run-rush.js...');
 
-        terminal.writeLine('The Rush engine has not been installed yet. Invoking install-run-rush.js...');
+          const installAndRunRushProcess: SpawnSyncReturns<string> = Executable.spawnSync(
+            'node',
+            [installAndRunRushJSPath, '--help'],
+            {
+              stdio: 'pipe'
+            }
+          );
 
-        const installAndRunRushProcess: SpawnSyncReturns<string> = Executable.spawnSync(
-          'node',
-          [installAndRunRushJSPath, '--help'],
-          {
-            stdio: 'pipe'
+          installAndRunRushStderrContent = installAndRunRushProcess.stderr;
+          if (installAndRunRushProcess.status !== 0) {
+            throw new Error(`The ${RUSH_LIB_NAME} package failed to install`);
           }
-        );
 
-        installAndRunRushStderrContent = installAndRunRushProcess.stderr;
-        if (installAndRunRushProcess.status !== 0) {
-          throw new Error(`The ${RUSH_LIB_NAME} package failed to install`);
+          // Retry to load "rush-lib" after install-run-rush run
+          terminal.writeVerboseLine(
+            `Trying to load  ${RUSH_LIB_NAME} installed by install-run-rush a second time`
+          );
+          sdkContext.rushLibModule = requireRushLibUnderFolderPath(installRunNodeModuleFolder);
+        } catch (e2) {
+          // eslint-disable-next-line no-console
+          console.error(`${installAndRunRushStderrContent}`);
+          throw new Error(`The ${RUSH_LIB_NAME} package failed to load`);
         }
-
-        // Retry to load "rush-lib" after install-run-rush run
-        terminal.writeVerboseLine(
-          `Trying to load  ${RUSH_LIB_NAME} installed by install-run-rush a second time`
-        );
-        sdkContext.rushLibModule = requireRushLibUnderFolderPath(installRunNodeModuleFolder);
-      } catch (e2) {
-        // eslint-disable-next-line no-console
-        console.error(`${installAndRunRushStderrContent}`);
-        throw new Error(`The ${RUSH_LIB_NAME} package failed to load`);
       }
-    }
 
-    if (sdkContext.rushLibModule !== undefined) {
-      // to track which scenario is active and how it got initialized.
-      global.___rush___rushLibModuleFromInstallAndRunRush = sdkContext.rushLibModule;
-      terminal.writeVerboseLine(`Loaded ${RUSH_LIB_NAME} installed by install-run-rush`);
+      if (sdkContext.rushLibModule !== undefined) {
+        // to track which scenario is active and how it got initialized.
+        global.___rush___rushLibModuleFromInstallAndRunRush = sdkContext.rushLibModule;
+        terminal.writeVerboseLine(`Loaded ${RUSH_LIB_NAME} installed by install-run-rush`);
+      }
     }
   } catch (e) {
     // no-catch

libraries/rush-sdk/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "./node_modules/local-node-rig/profiles/default/tsconfig-base.json",
   "compilerOptions": {
-    "outDir": "lib-shim",
+    "outDir": "lib-commonjs",
     "types": [
       "node",
       "heft-jest",

libraries/rush-sdk/webpack.config.js

@@ -0,0 +1,71 @@
+/* eslint-env es6 */
+'use strict';
+
+const webpack = require('webpack');
+const { PackageJsonLookup } = require('@rushstack/node-core-library');
+const { PreserveDynamicRequireWebpackPlugin } = require('@rushstack/webpack-preserve-dynamic-require-plugin');
+
+module.exports = () => {
+  const packageJson = PackageJsonLookup.loadOwnPackageJson(__dirname);
+
+  const externalDependencyNames = new Set([...Object.keys(packageJson.dependencies || {})]);
+
+  // Explicitly exclude @microsoft/rush-lib
+  externalDependencyNames.delete('@microsoft/rush-lib');
+
+  return {
+    context: __dirname,
+    mode: 'development', // So the output isn't minified
+    devtool: 'source-map',
+    entry: {
+      index: `${__dirname}/lib-commonjs/index.js`,
+      loader: `${__dirname}/lib-commonjs/loader.js`
+    },
+    output: {
+      path: `${__dirname}/lib-shim`,
+      filename: '[name].js',
+      chunkFilename: 'chunks/[name].js',
+      library: {
+        type: 'commonjs2'
+      }
+    },
+    target: 'node',
+    plugins: [new PreserveDynamicRequireWebpackPlugin()],
+    externals: [
+      ({ request }, callback) => {
+        let packageName;
+        let firstSlashIndex = request.indexOf('/');
+        if (firstSlashIndex === -1) {
+          packageName = request;
+        } else if (request.startsWith('@')) {
+          let secondSlash = request.indexOf('/', firstSlashIndex + 1);
+          if (secondSlash === -1) {
+            packageName = request;
+          } else {
+            packageName = request.substring(0, secondSlash);
+          }
+        } else {
+          packageName = request.substring(0, firstSlashIndex);
+        }
+
+        if (externalDependencyNames.has(packageName)) {
+          callback(null, `commonjs ${request}`);
+        } else {
+          callback();
+        }
+      }
+    ],
+    optimization: {
+      splitChunks: {
+        chunks: 'all',
+        minChunks: 1,
+        cacheGroups: {
+          commons: {
+            name: 'commons',
+            chunks: 'initial'
+          }
+        }
+      }
+    }
+  };
+};