Skip to content

Commit 6215e24

Browse files
ido-plutoido-pluto
committed
feat(downloadURL): regenerate download url when token expires
1 parent c31e008 commit 6215e24

13 files changed

+178
-28
lines changed

‎package-lock.json‎

Lines changed: 42 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
"@types/fs-extra": "^11.0.1",
105105
"@types/lodash.debounce": "^4.0.9",
106106
"@types/node": "^20.4.9",
107+
"@types/stream-throttle": "^0.1.4",
107108
"@typescript-eslint/eslint-plugin": "^6.3.0",
108109
"@typescript-eslint/parser": "^6.3.0",
109110
"@vitest/ui": "^1.6.0",
@@ -117,6 +118,7 @@
117118
"hash.js": "^1.1.7",
118119
"husky": "^8.0.3",
119120
"semantic-release": "^24.0.0",
121+
"stream-throttle": "^0.1.3",
120122
"tslib": "^2.6.1",
121123
"typedoc": "^0.26.3",
122124
"typedoc-material-theme": "^1.1.0",

‎src/download/download-engine/download-file/download-engine-file.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,9 +278,7 @@ export default class DownloadEngineFile extends EventEmitter<DownloadEngineFileE
278278
startChunk,
279279
endChunk: endChunk,
280280
lastChunkEndsFile: endChunk === Infinity || endChunk === this._progress.chunks.length,
281-
totalSize: this._activePart.size,
282-
url: this._activePart.downloadURL!,
283-
rangeSupport: this._activePart.acceptRange,
281+
activePart: this._activePart,
284282
onProgress: (length: number) => {
285283
getContext().streamBytes = length;
286284
this._sendProgressDownloadPart();

‎src/download/download-engine/engine/base-download-engine.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ export type BaseDownloadEngineEvents = {
3838
[key: string]: any
3939
};
4040

41+
export const DEFAULT_BASE_DOWNLOAD_ENGINE_OPTIONS: Partial<BaseDownloadEngineOptions> = {
42+
reuseRedirectURL: true
43+
};
44+
4145
export default class BaseDownloadEngine extends EventEmitter<BaseDownloadEngineEvents> {
4246
public readonly options: DownloadEngineFileOptions;
4347
protected readonly _engine: DownloadEngineFile;
@@ -165,6 +169,8 @@ export default class BaseDownloadEngine extends EventEmitter<BaseDownloadEngineE
165169

166170
return {
167171
downloadURL,
172+
originalURL: part,
173+
downloadURLUpdateDate: Date.now(),
168174
size,
169175
acceptRange: size > 0 && acceptRange
170176
};
@@ -173,6 +179,8 @@ export default class BaseDownloadEngine extends EventEmitter<BaseDownloadEngineE
173179
// if the server does not support HEAD request, we will skip that step
174180
return {
175181
downloadURL: part,
182+
originalURL: part,
183+
downloadURLUpdateDate: Date.now(),
176184
size: 0,
177185
acceptRange: false
178186
};

‎src/download/download-engine/engine/download-engine-browser.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import DownloadEngineFile from "../download-file/download-engine-file.js";
33
import DownloadEngineFetchStreamFetch from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js";
44
import DownloadEngineFetchStreamXhr from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.js";
55
import DownloadEngineWriteStreamBrowser, {DownloadEngineWriteStreamBrowserWriter} from "../streams/download-engine-write-stream/download-engine-write-stream-browser.js";
6-
import BaseDownloadEngine, {BaseDownloadEngineOptions} from "./base-download-engine.js";
6+
import BaseDownloadEngine, {BaseDownloadEngineOptions, DEFAULT_BASE_DOWNLOAD_ENGINE_OPTIONS} from "./base-download-engine.js";
77
import BaseDownloadEngineWriteStream from "../streams/download-engine-write-stream/base-download-engine-write-stream.js";
88
import BaseDownloadEngineFetchStream from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
99

@@ -44,6 +44,8 @@ export default class DownloadEngineBrowser<WriteStream extends BaseDownloadEngin
4444
* Download file
4545
*/
4646
public static async createFromOptions(options: DownloadEngineOptionsBrowser) {
47+
options = Object.assign({}, DEFAULT_BASE_DOWNLOAD_ENGINE_OPTIONS, options);
48+
4749
DownloadEngineBrowser._validateOptions(options);
4850
const partURLs = "partURLs" in options ? options.partURLs : [options.url];
4951

‎src/download/download-engine/engine/download-engine-nodejs.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import DownloadEngineFile from "../download-file/download-engine-file.js";
44
import DownloadEngineFetchStreamFetch from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js";
55
import DownloadEngineWriteStreamNodejs from "../streams/download-engine-write-stream/download-engine-write-stream-nodejs.js";
66
import DownloadEngineFetchStreamLocalFile from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.js";
7-
import BaseDownloadEngine, {BaseDownloadEngineOptions} from "./base-download-engine.js";
7+
import BaseDownloadEngine, {BaseDownloadEngineOptions, DEFAULT_BASE_DOWNLOAD_ENGINE_OPTIONS} from "./base-download-engine.js";
88
import SavePathError from "./error/save-path-error.js";
99
import fs from "fs-extra";
1010
import BaseDownloadEngineFetchStream from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
@@ -134,6 +134,8 @@ export default class DownloadEngineNodejs<T extends DownloadEngineWriteStreamNod
134134
* By default, it will guess the strategy based on the URL
135135
*/
136136
public static async createFromOptions(options: DownloadEngineOptionsNodejs) {
137+
options = Object.assign({}, DEFAULT_BASE_DOWNLOAD_ENGINE_OPTIONS, options);
138+
137139
DownloadEngineNodejs._validateOptions(options);
138140
const partURLs = "partURLs" in options ? options.partURLs : [options.url];
139141

@@ -146,7 +148,7 @@ export default class DownloadEngineNodejs<T extends DownloadEngineWriteStreamNod
146148
}
147149

148150
protected static async _createFromOptionsWithCustomFetch(options: DownloadEngineOptionsNodejsCustomFetch) {
149-
const downloadFile = await DownloadEngineNodejs._createDownloadFile(options.partURLs, options.fetchStream);
151+
const downloadFile = await DownloadEngineNodejs._createDownloadFile(options.partURLs, options.fetchStream, options);
150152
const downloadLocation = DownloadEngineNodejs._createDownloadLocation(downloadFile, options);
151153
downloadFile.localFileName = path.basename(downloadLocation);
152154

‎src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ import {AvailablePrograms} from "../../download-file/download-programs/switch-pr
55
import HttpError from "./errors/http-error.js";
66
import StatusCodeError from "./errors/status-code-error.js";
77
import sleep from "sleep-promise";
8+
import {withLock} from "lifecycle-utils";
89

910
export const STREAM_NOT_RESPONDING_TIMEOUT = 1000 * 3;
10-
1111
export const MIN_LENGTH_FOR_MORE_INFO_REQUEST = 1024 * 1024 * 3; // 3MB
1212

13+
const TOKEN_EXPIRED_ERROR_CODES = [401, 403, 419, 440, 498, 499];
14+
1315
export type BaseDownloadEngineFetchStreamOptions = {
1416
retry?: retry.Options
1517
retryFetchDownloadInfo?: retry.Options
@@ -50,14 +52,18 @@ export type DownloadInfoResponse = {
5052
};
5153

5254
export type FetchSubState = {
53-
url: string,
5455
startChunk: number,
5556
endChunk: number,
5657
lastChunkEndsFile: boolean,
57-
totalSize: number,
5858
chunkSize: number,
59-
rangeSupport?: boolean,
6059
onProgress?: (length: number) => void,
60+
activePart: {
61+
size: number,
62+
acceptRange?: boolean,
63+
downloadURL: string,
64+
originalURL: string,
65+
downloadURLUpdateDate: number
66+
}
6167
};
6268

6369
export type BaseDownloadEngineFetchStreamEvents = {
@@ -102,6 +108,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter
102108
public aborted = false;
103109
protected _pausedResolve?: () => void;
104110
public errorCount = {value: 0};
111+
public lastFetchTime = 0;
105112

106113
constructor(options: Partial<BaseDownloadEngineFetchStreamOptions> = {}) {
107114
super();
@@ -114,7 +121,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter
114121
}
115122

116123
protected get _endSize() {
117-
return Math.min(this.state.endChunk * this.state.chunkSize, this.state.totalSize);
124+
return Math.min(this.state.endChunk * this.state.chunkSize, this.state.activePart.size);
118125
}
119126

120127
protected initEvents() {
@@ -210,6 +217,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter
210217
// eslint-disable-next-line no-constant-condition
211218
while (true) {
212219
try {
220+
this.lastFetchTime = Date.now();
213221
return await this.fetchWithoutRetryChunks((...args) => {
214222
if (retryingOn) {
215223
retryingOn = false;
@@ -219,10 +227,12 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter
219227
});
220228
} catch (error: any) {
221229
if (error?.name === "AbortError") return;
230+
222231
this.errorCount.value++;
223232
this.emit("errorCountIncreased", this.errorCount.value, error);
224233

225-
if (error instanceof HttpError && !this.retryOnServerError(error)) {
234+
const needToRecreateURL = this.shouldRecreateURL(error);
235+
if (!needToRecreateURL && error instanceof HttpError && !this.retryOnServerError(error)) {
226236
throw error;
227237
}
228238

@@ -238,11 +248,31 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter
238248
retryResolvers = retryAsyncStatementSimple(this.options.retry);
239249
}
240250

241-
await retryResolvers(error);
251+
await Promise.all([
252+
retryResolvers(error),
253+
needToRecreateURL && this.recreateDownloadURL()
254+
]);
242255
}
243256
}
244257
}
245258

259+
shouldRecreateURL(error: Error): boolean {
260+
return error instanceof StatusCodeError && TOKEN_EXPIRED_ERROR_CODES.includes(error.statusCode) &&
261+
this.state.activePart.downloadURL !== this.state.activePart.originalURL;
262+
}
263+
264+
recreateDownloadURL() {
265+
return withLock(this.state.activePart, "_recreateURLLock", async () => {
266+
if (this.state.activePart.downloadURLUpdateDate > this.lastFetchTime) {
267+
return; // The URL was updated while we were waiting for the lock
268+
}
269+
270+
const downloadInfo = await this.fetchDownloadInfo(this.state.activePart.originalURL);
271+
this.state.activePart.downloadURL = downloadInfo.newURL || this.state.activePart.originalURL;
272+
this.state.activePart.downloadURLUpdateDate = Date.now();
273+
});
274+
}
275+
246276
protected abstract fetchWithoutRetryChunks(callback: WriteCallback): Promise<void> | void;
247277

248278
public close(): void | Promise<void> {

‎src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe
3232
...this.options.headers
3333
};
3434

35-
if (this.state.rangeSupport) {
35+
if (this.state.activePart.acceptRange) {
3636
headers.range = `bytes=${this._startSize}-${this._endSize - 1}`;
3737
}
3838

@@ -49,18 +49,18 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe
4949
});
5050

5151

52-
response = await fetch(this.appendToURL(this.state.url), {
52+
response = await fetch(this.appendToURL(this.state.activePart.downloadURL), {
5353
headers,
5454
signal: this._activeController.signal
5555
});
5656

5757
if (response.status < 200 || response.status >= 300) {
58-
throw new StatusCodeError(this.state.url, response.status, response.statusText, headers);
58+
throw new StatusCodeError(this.state.activePart.downloadURL, response.status, response.statusText, headers);
5959
}
6060

6161
const contentLength = parseHttpContentRange(response.headers.get("content-range"))?.length ?? parseInt(response.headers.get("content-length")!);
6262
const expectedContentLength = this._endSize - this._startSize;
63-
if (this.state.rangeSupport && contentLength !== expectedContentLength) {
63+
if (this.state.activePart.acceptRange && contentLength !== expectedContentLength) {
6464
throw new InvalidContentLengthError(expectedContentLength, contentLength);
6565
}
6666

‎src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ export default class DownloadEngineFetchStreamLocalFile extends BaseDownloadEngi
3333
}
3434

3535
protected override async fetchWithoutRetryChunks(callback: WriteCallback): Promise<void> {
36-
const file = await this._ensureFileOpen(this.state.url);
36+
const file = await this._ensureFileOpen(this.state.activePart.downloadURL);
3737

3838
const stream = file.createReadStream({
3939
start: this._startSize,

‎src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
4242
...this.options.headers
4343
};
4444

45-
if (this.state.rangeSupport) {
45+
if (this.state.activePart.acceptRange) {
4646
headers.range = `bytes=${start}-${end - 1}`;
4747
}
4848

@@ -90,7 +90,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
9090
clearStreamTimeout();
9191
const contentLength = parseInt(xhr.getResponseHeader("content-length")!);
9292

93-
if (this.state.rangeSupport && contentLength !== end - start) {
93+
if (this.state.activePart.acceptRange && contentLength !== end - start) {
9494
throw new InvalidContentLengthError(end - start, contentLength);
9595
}
9696

@@ -133,7 +133,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
133133
}
134134

135135
public override async fetchChunks(callback: WriteCallback) {
136-
if (this.state.rangeSupport) {
136+
if (this.state.activePart.acceptRange) {
137137
return await this._fetchChunksRangeSupport(callback);
138138
}
139139

@@ -149,14 +149,14 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
149149
await this.paused;
150150
if (this.aborted) return;
151151

152-
const chunk = await this.fetchBytes(this.state.url, this._startSize, this._endSize, this.state.onProgress);
152+
const chunk = await this.fetchBytes(this.state.activePart.downloadURL, this._startSize, this._endSize, this.state.onProgress);
153153
callback([chunk], this._startSize, this.state.startChunk++);
154154
}
155155
}
156156

157157
protected async _fetchChunksWithoutRange(callback: WriteCallback) {
158158
const relevantContent = await (async (): Promise<Uint8Array> => {
159-
const result = await this.fetchBytes(this.state.url, 0, this._endSize, this.state.onProgress);
159+
const result = await this.fetchBytes(this.state.activePart.downloadURL, 0, this._endSize, this.state.onProgress);
160160
return result.slice(this._startSize, this._endSize || result.length);
161161
})();
162162

0 commit comments

Comments
 (0)