From bd99ab223205630eae5c8eab977301653cb0c712 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 5 Dec 2024 13:29:21 -0500 Subject: [PATCH 01/12] Remove node-fetch. --- .../rush/main_2024-12-05-18-29.json | 10 + .../build-tests-subspace/pnpm-lock.yaml | 1 - .../build-tests-subspace/repo-state.json | 4 +- .../config/subspaces/default/pnpm-lock.yaml | 9 - libraries/rush-lib/package.json | 2 - .../src/logic/base/BaseInstallManager.ts | 10 +- .../src/logic/setup/SetupPackageRegistry.ts | 6 +- libraries/rush-lib/src/utilities/WebClient.ts | 143 ++++-- .../src/utilities/test/WebClient.test.ts | 64 +-- libraries/rush-sdk/package.json | 1 - .../src/AmazonS3Client.ts | 34 +- .../src/test/AmazonS3Client.test.ts | 16 +- .../__snapshots__/AmazonS3Client.test.ts.snap | 444 +++++------------- .../src/HttpBuildCacheProvider.ts | 10 +- 14 files changed, 302 insertions(+), 452 deletions(-) create mode 100644 common/changes/@microsoft/rush/main_2024-12-05-18-29.json diff --git a/common/changes/@microsoft/rush/main_2024-12-05-18-29.json b/common/changes/@microsoft/rush/main_2024-12-05-18-29.json new file mode 100644 index 00000000000..342ca115335 --- /dev/null +++ b/common/changes/@microsoft/rush/main_2024-12-05-18-29.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Remove the dependency on node-fetch.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/config/subspaces/build-tests-subspace/pnpm-lock.yaml b/common/config/subspaces/build-tests-subspace/pnpm-lock.yaml index a33129d01a5..736e2e11b0c 100644 --- a/common/config/subspaces/build-tests-subspace/pnpm-lock.yaml +++ b/common/config/subspaces/build-tests-subspace/pnpm-lock.yaml @@ -6507,7 +6507,6 @@ packages: uuid: 8.3.2 transitivePeerDependencies: - '@types/node' - - encoding - supports-color file:../../../libraries/rush-sdk(@types/node@18.17.15): diff --git a/common/config/subspaces/build-tests-subspace/repo-state.json b/common/config/subspaces/build-tests-subspace/repo-state.json index 46eec8c7b13..d4724370fab 100644 --- a/common/config/subspaces/build-tests-subspace/repo-state.json +++ b/common/config/subspaces/build-tests-subspace/repo-state.json @@ -1,6 +1,6 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "be8fa6aee16239aa65766063b1b600566fe13d2b", + "pnpmShrinkwrapHash": "91719dffed0a42aec00fee22b00e6c236b96fc7e", "preferredVersionsHash": "ce857ea0536b894ec8f346aaea08cfd85a5af648", - "packageJsonInjectedDependenciesHash": "3e941f94bf991fc5ad180bc33d43dec969990e74" + "packageJsonInjectedDependenciesHash": "7bb5b296a95ab3932d6ffa5b97edd5ea79613f33" } diff --git a/common/config/subspaces/default/pnpm-lock.yaml b/common/config/subspaces/default/pnpm-lock.yaml index 3b86765e5d4..5a9bcb3e75c 100644 --- a/common/config/subspaces/default/pnpm-lock.yaml +++ b/common/config/subspaces/default/pnpm-lock.yaml @@ -3362,9 +3362,6 @@ importers: '@rushstack/ts-command-line': specifier: workspace:* version: link:../ts-command-line - '@types/node-fetch': - specifier: 2.6.2 - version: 2.6.2 '@yarnpkg/lockfile': specifier: ~1.0.2 version: 1.0.2 @@ -3401,9 +3398,6 @@ importers: js-yaml: specifier: ~3.13.1 version: 3.13.1 - node-fetch: - specifier: 2.6.7 - version: 2.6.7 npm-check: specifier: ~6.0.1 version: 6.0.1 @@ -3513,9 +3507,6 @@ importers: '@rushstack/terminal': specifier: workspace:* version: link:../terminal - '@types/node-fetch': - specifier: 2.6.2 - version: 2.6.2 tapable: specifier: 2.2.1 version: 2.2.1 diff --git a/libraries/rush-lib/package.json b/libraries/rush-lib/package.json index 4ecd59257c2..61053c29184 100644 --- a/libraries/rush-lib/package.json +++ b/libraries/rush-lib/package.json @@ -40,7 +40,6 @@ "@rushstack/stream-collator": "workspace:*", "@rushstack/terminal": "workspace:*", "@rushstack/ts-command-line": "workspace:*", - "@types/node-fetch": "2.6.2", "@yarnpkg/lockfile": "~1.0.2", "builtin-modules": "~3.1.0", "cli-table": "~0.3.1", @@ -53,7 +52,6 @@ "ignore": "~5.1.6", "inquirer": "~7.3.3", "js-yaml": "~3.13.1", - "node-fetch": "2.6.7", "npm-check": "~6.0.1", "npm-package-arg": "~6.1.0", "read-package-tree": "~5.1.5", diff --git a/libraries/rush-lib/src/logic/base/BaseInstallManager.ts b/libraries/rush-lib/src/logic/base/BaseInstallManager.ts index c0d081cc23b..1198d2c65a3 100644 --- a/libraries/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/libraries/rush-lib/src/logic/base/BaseInstallManager.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import type * as fetch from 'node-fetch'; import * as os from 'os'; import * as path from 'path'; import * as crypto from 'crypto'; @@ -48,7 +47,7 @@ import { ShrinkwrapFileFactory } from '../ShrinkwrapFileFactory'; import { Utilities } from '../../utilities/Utilities'; import { InstallHelpers } from '../installManager/InstallHelpers'; import * as PolicyValidator from '../policy/PolicyValidator'; -import type { WebClient as WebClientType, WebClientResponse } from '../../utilities/WebClient'; +import type { WebClient as WebClientType, IWebClientResponse } from '../../utilities/WebClient'; import { SetupPackageRegistry } from '../setup/SetupPackageRegistry'; import { PnpmfileConfiguration } from '../pnpm/PnpmfileConfiguration'; import type { IInstallManagerOptions } from './BaseInstallManagerTypes'; @@ -1066,12 +1065,13 @@ ${gitLfsHookHandling} webClient.userAgent = `pnpm/? npm/? node/${process.version} ${os.platform()} ${os.arch()}`; webClient.accept = 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*'; - const response: WebClientResponse = await webClient.fetchAsync(queryUrl); + const response: IWebClientResponse = await webClient.fetchAsync(queryUrl); if (!response.ok) { throw new Error('Failed to query'); } - const data: { versions: { [version: string]: { dist: { tarball: string } } } } = await response.json(); + const data: { versions: { [version: string]: { dist: { tarball: string } } } } = + await response.getJsonAsync(); let url: string; try { if (!data.versions[Rush.version]) { @@ -1090,7 +1090,7 @@ ${gitLfsHookHandling} // Make sure the tarball wasn't deleted from the CDN webClient.accept = '*/*'; - const response2: fetch.Response = await webClient.fetchAsync(url); + const response2: IWebClientResponse = await webClient.fetchAsync(url); if (!response2.ok) { if (response2.status === 404) { diff --git a/libraries/rush-lib/src/logic/setup/SetupPackageRegistry.ts b/libraries/rush-lib/src/logic/setup/SetupPackageRegistry.ts index ac6acbc2a43..7a5160d939e 100644 --- a/libraries/rush-lib/src/logic/setup/SetupPackageRegistry.ts +++ b/libraries/rush-lib/src/logic/setup/SetupPackageRegistry.ts @@ -17,7 +17,7 @@ import { PrintUtilities, Colorize, ConsoleTerminalProvider, Terminal } from '@ru import type { RushConfiguration } from '../../api/RushConfiguration'; import { Utilities } from '../../utilities/Utilities'; import { type IArtifactoryPackageRegistryJson, ArtifactoryConfiguration } from './ArtifactoryConfiguration'; -import type { WebClient as WebClientType, WebClientResponse } from '../../utilities/WebClient'; +import type { WebClient as WebClientType, IWebClientResponse } from '../../utilities/WebClient'; import { TerminalInput } from './TerminalInput'; interface IArtifactoryCustomizableMessages { @@ -300,7 +300,7 @@ export class SetupPackageRegistry { // our token. queryUrl += `auth/.npm`; - let response: WebClientResponse; + let response: IWebClientResponse; try { response = await webClient.fetchAsync(queryUrl); } catch (e) { @@ -324,7 +324,7 @@ export class SetupPackageRegistry { // //your-company.jfrog.io/your-artifacts/api/npm/npm-private/:username=your.name@your-company.com // //your-company.jfrog.io/your-artifacts/api/npm/npm-private/:email=your.name@your-company.com // //your-company.jfrog.io/your-artifacts/api/npm/npm-private/:always-auth=true - const responseText: string = await response.text(); + const responseText: string = await response.getTextAsync(); const responseLines: string[] = Text.convertToLf(responseText).trim().split('\n'); if (responseLines.length < 2 || !responseLines[0].startsWith('@.npm:')) { throw new Error('Unexpected response from Artifactory'); diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 78afdb74f82..8d3097c011b 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -3,8 +3,10 @@ import * as os from 'os'; import * as process from 'process'; -import * as fetch from 'node-fetch'; import type * as http from 'http'; +import { request as httpRequest, type IncomingMessage } from 'node:http'; +import { request as httpsRequest, type RequestOptions } from 'node:https'; +import type { Socket } from 'node:net'; import { Import } from '@rushstack/node-core-library'; const createHttpsProxyAgent: typeof import('https-proxy-agent') = Import.lazy('https-proxy-agent', require); @@ -12,22 +14,23 @@ const createHttpsProxyAgent: typeof import('https-proxy-agent') = Import.lazy('h /** * For use with {@link WebClient}. */ -export type WebClientResponse = fetch.Response; - -/** - * For use with {@link WebClient}. - */ -export type WebClientHeaders = fetch.Headers; -// eslint-disable-next-line @typescript-eslint/no-redeclare -export const WebClientHeaders: typeof fetch.Headers = fetch.Headers; +export interface IWebClientResponse { + ok: boolean; + status: number; + statusText?: string; + redirected: boolean; + getTextAsync: () => Promise; + getJsonAsync: () => Promise; + getBufferAsync: () => Promise; +} /** * For use with {@link WebClient}. */ export interface IWebFetchOptionsBase { timeoutMs?: number; - headers?: WebClientHeaders | Record; - redirect?: fetch.RequestInit['redirect']; + headers?: Record; + redirect?: 'follow' | 'error' | 'manual'; } /** @@ -53,49 +56,125 @@ export enum WebClientProxy { Detect, Fiddler } +export interface IRequestOptions extends RequestOptions, Pick {} + +export type FetchFn = ( + url: string, + options: IRequestOptions, + isRedirect?: boolean +) => Promise; + +const makeRequestAsync: FetchFn = async ( + url: string, + options: IRequestOptions, + redirected: boolean = false +) => { + const { body, redirect } = options; + + return await new Promise( + (resolve: (result: IWebClientResponse) => void, reject: (error: Error) => void) => { + const parsedUrl: URL = typeof url === 'string' ? new URL(url) : url; + const requestFunction: typeof httpRequest | typeof httpsRequest = + parsedUrl.protocol === 'https:' ? httpsRequest : httpRequest; + + requestFunction(url, options, (response: IncomingMessage) => { + const responseBuffers: (Buffer | Uint8Array)[] = []; + response.on('data', (chunk: string | Buffer | Uint8Array) => { + responseBuffers.push(Buffer.from(chunk)); + }); + response.on('end', () => { + // Handle retries by calling the method recursively with the redirect URL + const statusCode: number | undefined = response.statusCode; + if (statusCode === 301 || statusCode === 302) { + switch (redirect) { + case 'follow': { + const redirectUrl: string | string[] | undefined = response.headers.location; + if (redirectUrl) { + makeRequestAsync(redirectUrl, options, true).then(resolve).catch(reject); + } else { + reject( + new Error(`Received status code ${response.statusCode} with no location header: ${url}`) + ); + } + + break; + } + case 'error': + reject(new Error(`Received status code ${response.statusCode}: ${url}`)); + return; + } + } + + const responseData: Buffer = Buffer.concat(responseBuffers); + // const result: WebClientResponse = { response, responseData }; + const status: number = response.statusCode || 0; + const statusText: string | undefined = response.statusMessage; + const result: IWebClientResponse = { + ok: status >= 200 && status < 300, + status, + statusText, + redirected, + getTextAsync: async () => responseData.toString(), + getJsonAsync: async () => JSON.parse(responseData.toString()), + getBufferAsync: async () => responseData + }; + resolve(result); + }); + }) + .on('socket', (socket: Socket) => { + socket.on('error', (error: Error) => { + reject(error); + }); + }) + .on('error', (error: Error) => { + reject(error); + }) + .end(body); + } + ); +}; + +export const AUTHORIZATION_HEADER_NAME: 'Authorization' = 'Authorization'; +const ACCEPT_HEADER_NAME: 'accept' = 'accept'; +const USER_AGENT_HEADER_NAME: 'user-agent' = 'user-agent'; /** * A helper for issuing HTTP requests. */ export class WebClient { - private static _requestFn: typeof fetch.default = fetch.default; + private static _requestFn: FetchFn = makeRequestAsync; - public readonly standardHeaders: fetch.Headers = new fetch.Headers(); + public readonly standardHeaders: Record = {}; public accept: string | undefined = '*/*'; public userAgent: string | undefined = `rush node/${process.version} ${os.platform()} ${os.arch()}`; public proxy: WebClientProxy = WebClientProxy.Detect; - public static mockRequestFn(fn: typeof fetch.default): void { + public static mockRequestFn(fn: FetchFn): void { WebClient._requestFn = fn; } public static resetMockRequestFn(): void { - WebClient._requestFn = fetch.default; + WebClient._requestFn = makeRequestAsync; } - public static mergeHeaders(target: fetch.Headers, source: fetch.Headers | Record): void { - const iterator: Iterable<[string, string]> = - 'entries' in source && typeof source.entries === 'function' ? source.entries() : Object.entries(source); - - for (const [name, value] of iterator) { - target.set(name, value); + public static mergeHeaders(target: Record, source: Record): void { + for (const [name, value] of Object.entries(source)) { + target[name] = value; } } public addBasicAuthHeader(userName: string, password: string): void { - this.standardHeaders.set( - 'Authorization', - 'Basic ' + Buffer.from(userName + ':' + password).toString('base64') - ); + this.standardHeaders[AUTHORIZATION_HEADER_NAME] = + 'Basic ' + Buffer.from(userName + ':' + password).toString('base64'); } public async fetchAsync( url: string, options?: IGetFetchOptions | IFetchOptionsWithBody - ): Promise { - const headers: fetch.Headers = new fetch.Headers(); + ): Promise { + const headers: Record = {}; WebClient.mergeHeaders(headers, this.standardHeaders); @@ -104,11 +183,11 @@ export class WebClient { } if (this.userAgent) { - headers.set('user-agent', this.userAgent); + headers[USER_AGENT_HEADER_NAME] = this.userAgent; } if (this.accept) { - headers.set('accept', this.accept); + headers[ACCEPT_HEADER_NAME] = this.accept; } let proxyUrl: string = ''; @@ -136,10 +215,10 @@ export class WebClient { } const timeoutMs: number = options?.timeoutMs !== undefined ? options.timeoutMs : 15 * 1000; // 15 seconds - const requestInit: fetch.RequestInit = { + const requestInit: IRequestOptions = { method: options?.verb, - headers: headers, - agent: agent, + headers, + agent, timeout: timeoutMs, redirect: options?.redirect }; diff --git a/libraries/rush-lib/src/utilities/test/WebClient.test.ts b/libraries/rush-lib/src/utilities/test/WebClient.test.ts index ea968a7394c..59ef5e33ccc 100644 --- a/libraries/rush-lib/src/utilities/test/WebClient.test.ts +++ b/libraries/rush-lib/src/utilities/test/WebClient.test.ts @@ -1,89 +1,75 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { WebClient, WebClientHeaders } from '../WebClient'; +import { WebClient } from '../WebClient'; describe(WebClient.name, () => { describe(WebClient.mergeHeaders.name, () => { it('should merge headers', () => { - const target: WebClientHeaders = new WebClientHeaders({ header1: 'value1' }); - const source: WebClientHeaders = new WebClientHeaders({ header2: 'value2' }); + const target: Record = { header1: 'value1' }; + const source: Record = { header2: 'value2' }; WebClient.mergeHeaders(target, source); - expect(target.raw()).toMatchInlineSnapshot(` + expect(target).toMatchInlineSnapshot(` Object { - "header1": Array [ - "value1", - ], - "header2": Array [ - "value2", - ], + "header1": "value1", + "header2": "value2", } `); }); it('should handle an empty source', () => { - const target: WebClientHeaders = new WebClientHeaders({ header1: 'value1' }); - const source: WebClientHeaders = new WebClientHeaders(); + const target: Record = { header1: 'value1' }; + const source: Record = {}; WebClient.mergeHeaders(target, source); - expect(target.raw()).toMatchInlineSnapshot(` + expect(target).toMatchInlineSnapshot(` Object { - "header1": Array [ - "value1", - ], + "header1": "value1", } `); }); it('should handle an empty target', () => { - const target: WebClientHeaders = new WebClientHeaders(); - const source: WebClientHeaders = new WebClientHeaders({ header2: 'value2' }); + const target: Record = {}; + const source: Record = { header2: 'value2' }; WebClient.mergeHeaders(target, source); - expect(target.raw()).toMatchInlineSnapshot(` + expect(target).toMatchInlineSnapshot(` Object { - "header2": Array [ - "value2", - ], + "header2": "value2", } `); }); it('should handle both empty', () => { - const target: WebClientHeaders = new WebClientHeaders(); - const source: WebClientHeaders = new WebClientHeaders(); + const target: Record = {}; + const source: Record = {}; WebClient.mergeHeaders(target, source); - expect(target.raw()).toMatchInlineSnapshot(`Object {}`); + expect(target).toMatchInlineSnapshot(`Object {}`); }); it('should handle overwriting values', () => { - const target: WebClientHeaders = new WebClientHeaders({ header1: 'value1' }); - const source: WebClientHeaders = new WebClientHeaders({ header1: 'value2' }); + const target: Record = { header1: 'value1' }; + const source: Record = { header1: 'value2' }; WebClient.mergeHeaders(target, source); - expect(target.raw()).toMatchInlineSnapshot(` + expect(target).toMatchInlineSnapshot(` Object { - "header1": Array [ - "value2", - ], + "header1": "value2", } `); }); it('should handle a JS object as the source', () => { - const target: WebClientHeaders = new WebClientHeaders({ header1: 'value1' }); + const target: Record = { header1: 'value1' }; WebClient.mergeHeaders(target, { header2: 'value2' }); - expect(target.raw()).toMatchInlineSnapshot(` + expect(target).toMatchInlineSnapshot(` Object { - "header1": Array [ - "value1", - ], - "header2": Array [ - "value2", - ], + "header1": "value1", + "header2": "value2", } `); }); diff --git a/libraries/rush-sdk/package.json b/libraries/rush-sdk/package.json index d449ffa137b..46f2b3dd739 100644 --- a/libraries/rush-sdk/package.json +++ b/libraries/rush-sdk/package.json @@ -42,7 +42,6 @@ "@rushstack/node-core-library": "workspace:*", "@rushstack/package-deps-hash": "workspace:*", "@rushstack/terminal": "workspace:*", - "@types/node-fetch": "2.6.2", "tapable": "2.2.1" }, "devDependencies": { diff --git a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/AmazonS3Client.ts b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/AmazonS3Client.ts index 2894235df0c..384e33feb11 100644 --- a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/AmazonS3Client.ts +++ b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/AmazonS3Client.ts @@ -7,9 +7,9 @@ import * as crypto from 'crypto'; import { type IGetFetchOptions, type IFetchOptionsWithBody, - type WebClientResponse, + type IWebClientResponse, type WebClient, - WebClientHeaders + AUTHORIZATION_HEADER_NAME } from '@rushstack/rush-sdk/lib/utilities/WebClient'; import type { IAmazonS3BuildCacheProviderOptionsAdvanced } from './AmazonS3BuildCacheProvider'; @@ -118,11 +118,11 @@ export class AmazonS3Client { public async getObjectAsync(objectName: string): Promise { this._writeDebugLine('Reading object from S3'); return await this._sendCacheRequestWithRetriesAsync(async () => { - const response: WebClientResponse = await this._makeRequestAsync('GET', objectName); + const response: IWebClientResponse = await this._makeRequestAsync('GET', objectName); if (response.ok) { return { hasNetworkError: false, - response: await response.buffer() + response: await response.getBufferAsync() }; } else if (response.status === 404) { return { @@ -163,7 +163,7 @@ export class AmazonS3Client { } await this._sendCacheRequestWithRetriesAsync(async () => { - const response: WebClientResponse = await this._makeRequestAsync('PUT', objectName, objectBuffer); + const response: IWebClientResponse = await this._makeRequestAsync('PUT', objectName, objectBuffer); if (!response.ok) { return { hasNetworkError: true, @@ -199,12 +199,12 @@ export class AmazonS3Client { verb: 'GET' | 'PUT', objectName: string, body?: Buffer - ): Promise { + ): Promise { const isoDateString: IIsoDateString = this._getIsoDateString(); const bodyHash: string = this._getSha256(body); - const headers: WebClientHeaders = new WebClientHeaders(); - headers.set(DATE_HEADER_NAME, isoDateString.dateTime); - headers.set(CONTENT_HASH_HEADER_NAME, bodyHash); + const headers: Record = {}; + headers[DATE_HEADER_NAME] = isoDateString.dateTime; + headers[CONTENT_HASH_HEADER_NAME] = bodyHash; // the host can be e.g. https://s3.aws.com or http://localhost:9000 const host: string = this._s3Endpoint.replace(protocolRegex, ''); @@ -291,10 +291,10 @@ export class AmazonS3Client { const authorizationHeader: string = `AWS4-HMAC-SHA256 Credential=${this._credentials.accessKeyId}/${scope},SignedHeaders=${signedHeaderNamesString},Signature=${signature}`; - headers.set('Authorization', authorizationHeader); + headers[AUTHORIZATION_HEADER_NAME] = authorizationHeader; if (this._credentials.sessionToken) { // Handle signing with temporary credentials (via sts:assume-role) - headers.set('X-Amz-Security-Token', this._credentials.sessionToken); + headers['X-Amz-Security-Token'] = this._credentials.sessionToken; } } @@ -311,11 +311,11 @@ export class AmazonS3Client { this._writeDebugLine(Colorize.bold(Colorize.underline('Sending request to S3'))); this._writeDebugLine(Colorize.bold('HOST: '), url); this._writeDebugLine(Colorize.bold('Headers: ')); - headers.forEach((value, name) => { + for (const [name, value] of Object.entries(headers)) { this._writeDebugLine(Colorize.cyan(`\t${name}: ${value}`)); - }); + } - const response: WebClientResponse = await this._webClient.fetchAsync(url, webFetchOptions); + const response: IWebClientResponse = await this._webClient.fetchAsync(url, webFetchOptions); return response; } @@ -356,16 +356,16 @@ export class AmazonS3Client { }; } - private async _safeReadResponseTextAsync(response: WebClientResponse): Promise { + private async _safeReadResponseTextAsync(response: IWebClientResponse): Promise { try { - return await response.text(); + return await response.getTextAsync(); } catch (err) { // ignore the error } return undefined; } - private async _getS3ErrorAsync(response: WebClientResponse): Promise { + private async _getS3ErrorAsync(response: IWebClientResponse): Promise { const text: string | undefined = await this._safeReadResponseTextAsync(response); return new Error( `Amazon S3 responded with status code ${response.status} (${response.statusText})${ diff --git a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts index 1cdedf55015..24b5b6a9c6d 100644 --- a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts +++ b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts @@ -253,13 +253,23 @@ describe(AmazonS3Client.name, () => { response: IResponseOptions, testOptions: ITestOptions ): Promise { + const body: string | undefined = response.body; const spy: jest.SpyInstance = jest.spyOn(WebClient.prototype, 'fetchAsync').mockReturnValue( Promise.resolve({ - buffer: response.body ? async () => Buffer.from(response.body || '') : undefined, + getBufferAsync: body + ? () => Promise.resolve(Buffer.from(body)) + : () => Promise.reject(new Error('No body provided')), + getTextAsync: body + ? () => Promise.resolve(body) + : () => Promise.reject(new Error('No body provided')), + getJsonAsync: body + ? () => Promise.resolve(JSON.parse(body)) + : () => Promise.reject(new Error('No body provided')), status: response.status, statusText: response.statusText, - ok: response.status >= 200 && response.status < 300 - }) as unknown as ReturnType + ok: response.status >= 200 && response.status < 300, + redirected: false + }) ); const s3Client: AmazonS3Client = new AmazonS3Client(credentials, options, webClient, terminal); diff --git a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/__snapshots__/AmazonS3Client.test.ts.snap b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/__snapshots__/AmazonS3Client.test.ts.snap index 46526b4cfda..9481b3d6983 100644 --- a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/__snapshots__/AmazonS3Client.test.ts.snap +++ b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/__snapshots__/AmazonS3Client.test.ts.snap @@ -6,18 +6,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials Can g Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -28,18 +20,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials Can g Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=fa88dc2c0877d83d442298fd51281eeffa1196436397832a794b89a079302d71", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=fa88dc2c0877d83d442298fd51281eeffa1196436397832a794b89a079302d71", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -50,18 +34,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials Handl Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -74,18 +50,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials Handl Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -96,18 +64,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials Handl Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -120,21 +80,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -145,21 +95,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=c3e1597c716ad0146d1a0eca844194b3120f8d8f07cf0a2c402f0b4f2148de12", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=c3e1597c716ad0146d1a0eca844194b3120f8d8f07cf0a2c402f0b4f2148de12", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -170,21 +110,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -197,21 +127,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -222,21 +142,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -249,21 +159,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -276,21 +176,11 @@ exports[`AmazonS3Client Making requests Getting an object With credentials inclu Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=2a4b8f28b1bdd37af6bb0fd79212c223e2c28e4f94bfb5d1c94a16bb056d5624", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -303,18 +193,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials shoul Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -327,18 +209,10 @@ exports[`AmazonS3Client Making requests Getting an object With credentials shoul Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", - ], - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=194608e9e7ba6d8aa4a019b3b6fd237e6b09ef1f45ff7fa60cbb81c1875538be", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -351,15 +225,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ca Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -370,15 +238,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ca Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -389,15 +251,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ha Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -408,15 +264,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ha Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -429,15 +279,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ha Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -448,15 +292,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ha Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -467,15 +305,9 @@ exports[`AmazonS3Client Making requests Getting an object Without credentials Ha Array [ "http://localhost:9000/abc123", Object { - "headers": Headers { - Symbol(map): Object { - "x-amz-content-sha256": Array [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20200418T123242Z", }, "verb": "GET", }, @@ -508,18 +340,10 @@ Array [ ], "type": "Buffer", }, - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=1255739559ef86e1cc1e733fa9e13aa4990c6f1fb1ae821653540d401c48c5e1", - ], - "x-amz-content-sha256": Array [ - "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=1255739559ef86e1cc1e733fa9e13aa4990c6f1fb1ae821653540d401c48c5e1", + "x-amz-content-sha256": "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + "x-amz-date": "20200418T123242Z", }, "verb": "PUT", }, @@ -552,18 +376,10 @@ Array [ ], "type": "Buffer", }, - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=1255739559ef86e1cc1e733fa9e13aa4990c6f1fb1ae821653540d401c48c5e1", - ], - "x-amz-content-sha256": Array [ - "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=1255739559ef86e1cc1e733fa9e13aa4990c6f1fb1ae821653540d401c48c5e1", + "x-amz-content-sha256": "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + "x-amz-date": "20200418T123242Z", }, "verb": "PUT", }, @@ -594,18 +410,10 @@ Array [ ], "type": "Buffer", }, - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=66e311b6c1987dd62cb3bfec416b9129c966d1d15075d1ebf852433062bf4281", - ], - "x-amz-content-sha256": Array [ - "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=66e311b6c1987dd62cb3bfec416b9129c966d1d15075d1ebf852433062bf4281", + "x-amz-content-sha256": "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + "x-amz-date": "20200418T123242Z", }, "verb": "PUT", }, @@ -636,21 +444,11 @@ Array [ ], "type": "Buffer", }, - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=0ed8d55f5a3265d967092faf7e3ca7acd08ff3566651dc7c7363d60118c11528", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=0ed8d55f5a3265d967092faf7e3ca7acd08ff3566651dc7c7363d60118c11528", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + "x-amz-date": "20200418T123242Z", }, "verb": "PUT", }, @@ -683,21 +481,11 @@ Array [ ], "type": "Buffer", }, - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=0ed8d55f5a3265d967092faf7e3ca7acd08ff3566651dc7c7363d60118c11528", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=0ed8d55f5a3265d967092faf7e3ca7acd08ff3566651dc7c7363d60118c11528", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + "x-amz-date": "20200418T123242Z", }, "verb": "PUT", }, @@ -728,21 +516,11 @@ Array [ ], "type": "Buffer", }, - "headers": Headers { - Symbol(map): Object { - "Authorization": Array [ - "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=8e2cd7b6241780b51b1d15f428c43179d389b106be9a572e0def4cad6a5ba1e5", - ], - "X-Amz-Security-Token": Array [ - "sessionToken", - ], - "x-amz-content-sha256": Array [ - "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", - ], - "x-amz-date": Array [ - "20200418T123242Z", - ], - }, + "headers": Object { + "Authorization": "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=8e2cd7b6241780b51b1d15f428c43179d389b106be9a572e0def4cad6a5ba1e5", + "X-Amz-Security-Token": "sessionToken", + "x-amz-content-sha256": "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + "x-amz-date": "20200418T123242Z", }, "verb": "PUT", }, diff --git a/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts index f7ec565a154..4a6368b998d 100644 --- a/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts +++ b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts @@ -10,7 +10,7 @@ import { type RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk'; -import { WebClient, type WebClientResponse } from '@rushstack/rush-sdk/lib/utilities/WebClient'; +import { WebClient, type IWebClientResponse } from '@rushstack/rush-sdk/lib/utilities/WebClient'; import type { SpawnSyncReturns } from 'child_process'; enum CredentialsOptions { @@ -243,7 +243,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { terminal.writeDebugLine(`[http-build-cache] request: ${method} ${url} ${bodyLength} bytes`); const webClient: WebClient = new WebClient(); - const response: WebClientResponse = await webClient.fetchAsync(url, { + const response: IWebClientResponse = await webClient.fetchAsync(url, { verb: method, headers: headers, body: body, @@ -286,7 +286,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { return false; } - const result: Buffer | boolean = readBody ? Buffer.from(await response.arrayBuffer()) : true; + const result: Buffer | boolean = readBody ? await response.getBufferAsync() : true; terminal.writeDebugLine( `[http-build-cache] actual response: ${response.status} ${url} ${ @@ -351,7 +351,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { private _getFailureType( requestMethod: string, - response: WebClientResponse, + response: IWebClientResponse, isRedirect: boolean ): FailureType { if (response.ok) { @@ -403,7 +403,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { private _reportFailure( terminal: ITerminal, requestMethod: string, - response: WebClientResponse, + response: IWebClientResponse, isRedirect: boolean, message: string ): void { From a71727fe25f65282d2bf3dc48314342f6f49c1b8 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 5 Dec 2024 13:34:17 -0500 Subject: [PATCH 02/12] fixup! Remove node-fetch. --- libraries/rush-lib/src/utilities/WebClient.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 8d3097c011b..406424ee3ce 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -106,7 +106,6 @@ const makeRequestAsync: FetchFn = async ( } const responseData: Buffer = Buffer.concat(responseBuffers); - // const result: WebClientResponse = { response, responseData }; const status: number = response.statusCode || 0; const statusText: string | undefined = response.statusMessage; const result: IWebClientResponse = { From 17d3d48d208f621d13a216869a44053fa6379b2f Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 15:32:46 -0500 Subject: [PATCH 03/12] Make the next release of Rush a minor bump. --- common/config/rush/version-policies.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/config/rush/version-policies.json b/common/config/rush/version-policies.json index e0b053f84a9..bcffd8d873a 100644 --- a/common/config/rush/version-policies.json +++ b/common/config/rush/version-policies.json @@ -103,7 +103,7 @@ "policyName": "rush", "definitionName": "lockStepVersion", "version": "5.144.1", - "nextBump": "patch", + "nextBump": "minor", "mainProject": "@microsoft/rush" } ] From 85dfb01049d0963ce9f0b62a1a0ab832fd1af2b6 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 15:38:58 -0500 Subject: [PATCH 04/12] Convert inline snapshots to normal snapshots. --- .../src/utilities/test/WebClient.test.ts | 34 ++++-------------- .../test/__snapshots__/WebClient.test.ts.snap | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 28 deletions(-) create mode 100644 libraries/rush-lib/src/utilities/test/__snapshots__/WebClient.test.ts.snap diff --git a/libraries/rush-lib/src/utilities/test/WebClient.test.ts b/libraries/rush-lib/src/utilities/test/WebClient.test.ts index 59ef5e33ccc..cda35c0ee61 100644 --- a/libraries/rush-lib/src/utilities/test/WebClient.test.ts +++ b/libraries/rush-lib/src/utilities/test/WebClient.test.ts @@ -10,12 +10,7 @@ describe(WebClient.name, () => { const source: Record = { header2: 'value2' }; WebClient.mergeHeaders(target, source); - expect(target).toMatchInlineSnapshot(` -Object { - "header1": "value1", - "header2": "value2", -} -`); + expect(target).toMatchSnapshot(); }); it('should handle an empty source', () => { @@ -23,11 +18,7 @@ Object { const source: Record = {}; WebClient.mergeHeaders(target, source); - expect(target).toMatchInlineSnapshot(` -Object { - "header1": "value1", -} -`); + expect(target).toMatchSnapshot(); }); it('should handle an empty target', () => { @@ -35,11 +26,7 @@ Object { const source: Record = { header2: 'value2' }; WebClient.mergeHeaders(target, source); - expect(target).toMatchInlineSnapshot(` -Object { - "header2": "value2", -} -`); + expect(target).toMatchSnapshot(); }); it('should handle both empty', () => { @@ -47,7 +34,7 @@ Object { const source: Record = {}; WebClient.mergeHeaders(target, source); - expect(target).toMatchInlineSnapshot(`Object {}`); + expect(target).toMatchSnapshot(); }); it('should handle overwriting values', () => { @@ -55,23 +42,14 @@ Object { const source: Record = { header1: 'value2' }; WebClient.mergeHeaders(target, source); - expect(target).toMatchInlineSnapshot(` -Object { - "header1": "value2", -} -`); + expect(target).toMatchSnapshot(); }); it('should handle a JS object as the source', () => { const target: Record = { header1: 'value1' }; WebClient.mergeHeaders(target, { header2: 'value2' }); - expect(target).toMatchInlineSnapshot(` -Object { - "header1": "value1", - "header2": "value2", -} -`); + expect(target).toMatchSnapshot(); }); }); }); diff --git a/libraries/rush-lib/src/utilities/test/__snapshots__/WebClient.test.ts.snap b/libraries/rush-lib/src/utilities/test/__snapshots__/WebClient.test.ts.snap new file mode 100644 index 00000000000..82fdb7303c4 --- /dev/null +++ b/libraries/rush-lib/src/utilities/test/__snapshots__/WebClient.test.ts.snap @@ -0,0 +1,35 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`WebClient mergeHeaders should handle a JS object as the source 1`] = ` +Object { + "header1": "value1", + "header2": "value2", +} +`; + +exports[`WebClient mergeHeaders should handle an empty source 1`] = ` +Object { + "header1": "value1", +} +`; + +exports[`WebClient mergeHeaders should handle an empty target 1`] = ` +Object { + "header2": "value2", +} +`; + +exports[`WebClient mergeHeaders should handle both empty 1`] = `Object {}`; + +exports[`WebClient mergeHeaders should handle overwriting values 1`] = ` +Object { + "header1": "value2", +} +`; + +exports[`WebClient mergeHeaders should merge headers 1`] = ` +Object { + "header1": "value1", + "header2": "value2", +} +`; From 4562e4412ab171dea065f3b778b5c785c82e8eae Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 15:41:59 -0500 Subject: [PATCH 05/12] Memoize parsed and stringified response. --- libraries/rush-lib/src/utilities/WebClient.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 406424ee3ce..012da1e8dd8 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -108,13 +108,27 @@ const makeRequestAsync: FetchFn = async ( const responseData: Buffer = Buffer.concat(responseBuffers); const status: number = response.statusCode || 0; const statusText: string | undefined = response.statusMessage; + let bodyString: string | undefined; + let bodyJson: unknown | undefined; const result: IWebClientResponse = { ok: status >= 200 && status < 300, status, statusText, redirected, - getTextAsync: async () => responseData.toString(), - getJsonAsync: async () => JSON.parse(responseData.toString()), + getTextAsync: async () => { + if (bodyString === undefined) { + bodyString = responseData.toString(); + } + + return bodyString; + }, + getJsonAsync: async () => { + if (bodyJson === undefined) { + bodyJson = await result.getTextAsync(); + } + + return bodyJson as TJson; + }, getBufferAsync: async () => responseData }; resolve(result); From 5f09e6f5fca7a04a8e3a26eab4f4146dfae55604 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 16:00:04 -0500 Subject: [PATCH 06/12] Support compression. --- libraries/rush-lib/src/utilities/WebClient.ts | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 012da1e8dd8..b614106aff7 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -7,7 +7,7 @@ import type * as http from 'http'; import { request as httpRequest, type IncomingMessage } from 'node:http'; import { request as httpsRequest, type RequestOptions } from 'node:https'; import type { Socket } from 'node:net'; -import { Import } from '@rushstack/node-core-library'; +import { Import, LegacyAdapters } from '@rushstack/node-core-library'; const createHttpsProxyAgent: typeof import('https-proxy-agent') = Import.lazy('https-proxy-agent', require); @@ -19,6 +19,7 @@ export interface IWebClientResponse { status: number; statusText?: string; redirected: boolean; + headers: Record; getTextAsync: () => Promise; getJsonAsync: () => Promise; getBufferAsync: () => Promise; @@ -108,16 +109,21 @@ const makeRequestAsync: FetchFn = async ( const responseData: Buffer = Buffer.concat(responseBuffers); const status: number = response.statusCode || 0; const statusText: string | undefined = response.statusMessage; + const headers: Record = response.headers; + let bodyString: string | undefined; let bodyJson: unknown | undefined; + let decodedBuffer: Buffer | undefined; const result: IWebClientResponse = { ok: status >= 200 && status < 300, status, statusText, redirected, + headers, getTextAsync: async () => { if (bodyString === undefined) { - bodyString = responseData.toString(); + const buffer: Buffer = await result.getBufferAsync(); + bodyString = buffer.toString(); } return bodyString; @@ -129,7 +135,57 @@ const makeRequestAsync: FetchFn = async ( return bodyJson as TJson; }, - getBufferAsync: async () => responseData + getBufferAsync: async () => { + // Determine if the buffer is compressed and decode it if necessary + if (decodedBuffer === undefined) { + let encodings: string | string[] | undefined = headers['content-encoding']; + if (encodings !== undefined) { + const zlib: typeof import('zlib') = await import('zlib'); + if (!Array.isArray(encodings)) { + encodings = [encodings]; + } + + let buffer: Buffer = responseData; + for (const encoding of encodings) { + switch (encoding) { + case 'deflate': { + buffer = await LegacyAdapters.convertCallbackToPromise( + zlib.inflate.bind(zlib), + buffer + ); + break; + } + + case 'gzip': { + buffer = await LegacyAdapters.convertCallbackToPromise( + zlib.gunzip.bind(zlib), + buffer + ); + break; + } + + case 'br': { + buffer = await LegacyAdapters.convertCallbackToPromise( + zlib.brotliDecompress.bind(zlib), + buffer + ); + break; + } + + default: { + throw new Error(`Unsupported content-encoding: ${encodings}`); + } + } + } + + decodedBuffer = buffer; + } else { + decodedBuffer = responseData; + } + } + + return decodedBuffer; + } }; resolve(result); }); From 5007f8675631b1c35a7419e05581a93c8a9c1c9a Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 13:20:57 -0800 Subject: [PATCH 07/12] fixup! Support compression. --- libraries/rush-lib/src/utilities/WebClient.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index b614106aff7..40ba284d337 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -123,6 +123,7 @@ const makeRequestAsync: FetchFn = async ( getTextAsync: async () => { if (bodyString === undefined) { const buffer: Buffer = await result.getBufferAsync(); + // eslint-disable-next-line require-atomic-updates bodyString = buffer.toString(); } @@ -130,6 +131,7 @@ const makeRequestAsync: FetchFn = async ( }, getJsonAsync: async () => { if (bodyJson === undefined) { + // eslint-disable-next-line require-atomic-updates bodyJson = await result.getTextAsync(); } @@ -178,6 +180,7 @@ const makeRequestAsync: FetchFn = async ( } } + // eslint-disable-next-line require-atomic-updates decodedBuffer = buffer; } else { decodedBuffer = responseData; From 535f937cda2cec9981c36fef4dddb92bf42eb941 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 13:23:48 -0800 Subject: [PATCH 08/12] fixup! Support compression. --- .../src/test/AmazonS3Client.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts index 24b5b6a9c6d..07b5eddfc73 100644 --- a/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts +++ b/rush-plugins/rush-amazon-s3-build-cache-plugin/src/test/AmazonS3Client.test.ts @@ -265,6 +265,7 @@ describe(AmazonS3Client.name, () => { getJsonAsync: body ? () => Promise.resolve(JSON.parse(body)) : () => Promise.reject(new Error('No body provided')), + headers: {}, status: response.status, statusText: response.statusText, ok: response.status >= 200 && response.status < 300, From 377066623f7308d1a228045400dc9d97d7976ad6 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 13:40:09 -0800 Subject: [PATCH 09/12] Improve content decoding. --- libraries/rush-lib/src/utilities/WebClient.ts | 88 +++++++++++-------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 40ba284d337..93383c2129a 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -32,6 +32,14 @@ export interface IWebFetchOptionsBase { timeoutMs?: number; headers?: Record; redirect?: 'follow' | 'error' | 'manual'; + /** + * If true, the request will not include an Accept-Encoding header, and the response will not be decoded. + */ + noAcceptEncoding?: boolean; + /** + * If true, the response will not be decoded, but the Accept-Encoding header will still be sent. + */ + noDecode?: boolean; } /** @@ -57,7 +65,9 @@ export enum WebClientProxy { Detect, Fiddler } -export interface IRequestOptions extends RequestOptions, Pick {} +export interface IRequestOptions + extends RequestOptions, + Pick {} export type FetchFn = ( url: string, @@ -65,12 +75,21 @@ export type FetchFn = ( isRedirect?: boolean ) => Promise; +const DEFLATE_ENCODING: 'deflate' = 'deflate'; +const GZIP_ENCODING: 'gzip' = 'gzip'; +const BROTLI_ENCODING: 'br' = 'br'; +export const AUTHORIZATION_HEADER_NAME: 'Authorization' = 'Authorization'; +const ACCEPT_HEADER_NAME: 'accept' = 'accept'; +const USER_AGENT_HEADER_NAME: 'user-agent' = 'user-agent'; +const ACCEPT_ENCODING_HEADER_NAME: 'accept-encoding' = 'accept-encoding'; +const CONTENT_ENCODING_HEADER_NAME: 'content-encoding' = 'content-encoding'; + const makeRequestAsync: FetchFn = async ( url: string, options: IRequestOptions, redirected: boolean = false ) => { - const { body, redirect } = options; + const { body, redirect, noAcceptEncoding, noDecode } = options; return await new Promise( (resolve: (result: IWebClientResponse) => void, reject: (error: Error) => void) => { @@ -140,8 +159,8 @@ const makeRequestAsync: FetchFn = async ( getBufferAsync: async () => { // Determine if the buffer is compressed and decode it if necessary if (decodedBuffer === undefined) { - let encodings: string | string[] | undefined = headers['content-encoding']; - if (encodings !== undefined) { + let encodings: string | string[] | undefined = headers[CONTENT_ENCODING_HEADER_NAME]; + if (!noAcceptEncoding && !noDecode && encodings !== undefined) { const zlib: typeof import('zlib') = await import('zlib'); if (!Array.isArray(encodings)) { encodings = [encodings]; @@ -149,35 +168,26 @@ const makeRequestAsync: FetchFn = async ( let buffer: Buffer = responseData; for (const encoding of encodings) { + let decompressFn: (buffer: Buffer, callback: import('zlib').CompressCallback) => void; switch (encoding) { - case 'deflate': { - buffer = await LegacyAdapters.convertCallbackToPromise( - zlib.inflate.bind(zlib), - buffer - ); + case DEFLATE_ENCODING: { + decompressFn = zlib.inflate.bind(zlib); break; } - - case 'gzip': { - buffer = await LegacyAdapters.convertCallbackToPromise( - zlib.gunzip.bind(zlib), - buffer - ); + case GZIP_ENCODING: { + decompressFn = zlib.gunzip.bind(zlib); break; } - - case 'br': { - buffer = await LegacyAdapters.convertCallbackToPromise( - zlib.brotliDecompress.bind(zlib), - buffer - ); + case BROTLI_ENCODING: { + decompressFn = zlib.brotliDecompress.bind(zlib); break; } - default: { throw new Error(`Unsupported content-encoding: ${encodings}`); } } + + buffer = await LegacyAdapters.convertCallbackToPromise(decompressFn, buffer); } // eslint-disable-next-line require-atomic-updates @@ -206,10 +216,6 @@ const makeRequestAsync: FetchFn = async ( ); }; -export const AUTHORIZATION_HEADER_NAME: 'Authorization' = 'Authorization'; -const ACCEPT_HEADER_NAME: 'accept' = 'accept'; -const USER_AGENT_HEADER_NAME: 'user-agent' = 'user-agent'; - /** * A helper for issuing HTTP requests. */ @@ -246,12 +252,22 @@ export class WebClient { url: string, options?: IGetFetchOptions | IFetchOptionsWithBody ): Promise { + const { + headers: optionsHeaders, + timeoutMs = 15 * 1000, + verb, + redirect, + body, + noAcceptEncoding, + noDecode + } = (options as IFetchOptionsWithBody | undefined) ?? {}; + const headers: Record = {}; WebClient.mergeHeaders(headers, this.standardHeaders); - if (options?.headers) { - WebClient.mergeHeaders(headers, options.headers); + if (optionsHeaders) { + WebClient.mergeHeaders(headers, optionsHeaders); } if (this.userAgent) { @@ -262,6 +278,10 @@ export class WebClient { headers[ACCEPT_HEADER_NAME] = this.accept; } + if (!noAcceptEncoding) { + headers[ACCEPT_ENCODING_HEADER_NAME] = [DEFLATE_ENCODING, GZIP_ENCODING, BROTLI_ENCODING].join(', '); + } + let proxyUrl: string = ''; switch (this.proxy) { @@ -286,18 +306,16 @@ export class WebClient { agent = createHttpsProxyAgent(proxyUrl); } - const timeoutMs: number = options?.timeoutMs !== undefined ? options.timeoutMs : 15 * 1000; // 15 seconds const requestInit: IRequestOptions = { - method: options?.verb, + method: verb, headers, agent, timeout: timeoutMs, - redirect: options?.redirect + redirect, + body, + noDecode, + noAcceptEncoding }; - const optionsWithBody: IFetchOptionsWithBody | undefined = options as IFetchOptionsWithBody | undefined; - if (optionsWithBody?.body) { - requestInit.body = optionsWithBody.body; - } return await WebClient._requestFn(url, requestInit); } From 49eec6e465035809c6d0ac700275beeb279fb5c6 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 13:41:15 -0800 Subject: [PATCH 10/12] fixup! Memoize parsed and stringified response. --- libraries/rush-lib/src/utilities/WebClient.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 93383c2129a..4a6b3113084 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -150,8 +150,9 @@ const makeRequestAsync: FetchFn = async ( }, getJsonAsync: async () => { if (bodyJson === undefined) { + const text: string = await result.getTextAsync(); // eslint-disable-next-line require-atomic-updates - bodyJson = await result.getTextAsync(); + bodyJson = JSON.parse(text); } return bodyJson as TJson; From ad6196506f35722cf7a0ca68141e1e4af5d14d29 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 13:47:32 -0800 Subject: [PATCH 11/12] fixup! Support compression. --- libraries/rush-lib/src/utilities/WebClient.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 4a6b3113084..2c15520bc58 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -164,13 +164,13 @@ const makeRequestAsync: FetchFn = async ( if (!noAcceptEncoding && !noDecode && encodings !== undefined) { const zlib: typeof import('zlib') = await import('zlib'); if (!Array.isArray(encodings)) { - encodings = [encodings]; + encodings = encodings.split(','); } let buffer: Buffer = responseData; for (const encoding of encodings) { let decompressFn: (buffer: Buffer, callback: import('zlib').CompressCallback) => void; - switch (encoding) { + switch (encoding.trim()) { case DEFLATE_ENCODING: { decompressFn = zlib.inflate.bind(zlib); break; From ec8341b399786c03dca9f2214fb71b95de86b110 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 Dec 2024 15:15:48 -0800 Subject: [PATCH 12/12] Drop the Accept-Encoding header. --- libraries/rush-lib/src/utilities/WebClient.ts | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/libraries/rush-lib/src/utilities/WebClient.ts b/libraries/rush-lib/src/utilities/WebClient.ts index 2c15520bc58..45948ecbb57 100644 --- a/libraries/rush-lib/src/utilities/WebClient.ts +++ b/libraries/rush-lib/src/utilities/WebClient.ts @@ -33,11 +33,7 @@ export interface IWebFetchOptionsBase { headers?: Record; redirect?: 'follow' | 'error' | 'manual'; /** - * If true, the request will not include an Accept-Encoding header, and the response will not be decoded. - */ - noAcceptEncoding?: boolean; - /** - * If true, the response will not be decoded, but the Accept-Encoding header will still be sent. + * If true, the response will not be decoded if a Content-Encoding header is present. */ noDecode?: boolean; } @@ -67,7 +63,7 @@ export enum WebClientProxy { } export interface IRequestOptions extends RequestOptions, - Pick {} + Pick {} export type FetchFn = ( url: string, @@ -81,7 +77,6 @@ const BROTLI_ENCODING: 'br' = 'br'; export const AUTHORIZATION_HEADER_NAME: 'Authorization' = 'Authorization'; const ACCEPT_HEADER_NAME: 'accept' = 'accept'; const USER_AGENT_HEADER_NAME: 'user-agent' = 'user-agent'; -const ACCEPT_ENCODING_HEADER_NAME: 'accept-encoding' = 'accept-encoding'; const CONTENT_ENCODING_HEADER_NAME: 'content-encoding' = 'content-encoding'; const makeRequestAsync: FetchFn = async ( @@ -89,7 +84,7 @@ const makeRequestAsync: FetchFn = async ( options: IRequestOptions, redirected: boolean = false ) => { - const { body, redirect, noAcceptEncoding, noDecode } = options; + const { body, redirect, noDecode } = options; return await new Promise( (resolve: (result: IWebClientResponse) => void, reject: (error: Error) => void) => { @@ -161,7 +156,7 @@ const makeRequestAsync: FetchFn = async ( // Determine if the buffer is compressed and decode it if necessary if (decodedBuffer === undefined) { let encodings: string | string[] | undefined = headers[CONTENT_ENCODING_HEADER_NAME]; - if (!noAcceptEncoding && !noDecode && encodings !== undefined) { + if (!noDecode && encodings !== undefined) { const zlib: typeof import('zlib') = await import('zlib'); if (!Array.isArray(encodings)) { encodings = encodings.split(','); @@ -259,7 +254,6 @@ export class WebClient { verb, redirect, body, - noAcceptEncoding, noDecode } = (options as IFetchOptionsWithBody | undefined) ?? {}; @@ -279,10 +273,6 @@ export class WebClient { headers[ACCEPT_HEADER_NAME] = this.accept; } - if (!noAcceptEncoding) { - headers[ACCEPT_ENCODING_HEADER_NAME] = [DEFLATE_ENCODING, GZIP_ENCODING, BROTLI_ENCODING].join(', '); - } - let proxyUrl: string = ''; switch (this.proxy) { @@ -314,8 +304,7 @@ export class WebClient { timeout: timeoutMs, redirect, body, - noDecode, - noAcceptEncoding + noDecode }; return await WebClient._requestFn(url, requestInit);