Skip to content

Commit 748d793

Browse files
chriswkchriswk
authored
feat: Backoff based on HTTP statuses (#537)
New behaviour: * Will emit error event and stop posting metrics/polling features on 401,403 and 404 * Will emit warn event and back-off to 2,3,4,5,6,7,8,9,10 * normal poll/metrics interval on 429,500,502,503 and 504.
1 parent 8aefb0e commit 748d793

File tree

8 files changed

+545
-126
lines changed

8 files changed

+545
-126
lines changed

‎src/events.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export enum UnleashEvents {
1212
CountVariant = 'countVariant',
1313
Sent = 'sent',
1414
Registered = 'registered',
15-
Impression = 'impression'
15+
Impression = 'impression',
1616
}
1717

1818
export interface ImpressionEvent {

‎src/metrics.ts

Lines changed: 67 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@ interface MetricsData {
3838
}
3939

4040
interface RegistrationData {
41-
appName: string;
41+
appName: string;
4242
instanceId: string;
4343
sdkVersion: string;
4444
strategies: string[];
45-
started: Date;
46-
interval: number
45+
started: Date;
46+
interval: number;
4747
}
4848

4949
export default class Metrics extends EventEmitter {
@@ -61,6 +61,8 @@ export default class Metrics extends EventEmitter {
6161

6262
private metricsJitter: number;
6363

64+
private failures: number = 0;
65+
6466
private disabled: boolean;
6567

6668
private url: string;
@@ -111,21 +113,39 @@ export default class Metrics extends EventEmitter {
111113
return getAppliedJitter(this.metricsJitter);
112114
}
113115

116+
getFailures(): number {
117+
return this.failures;
118+
}
119+
120+
getInterval(): number {
121+
if(this.metricsInterval === 0) {
122+
return 0;
123+
} else {
124+
return this.metricsInterval +
125+
(this.failures * this.metricsInterval) +
126+
this.getAppliedJitter();
127+
}
128+
129+
}
130+
114131
private startTimer(): void {
115-
if (this.disabled) {
132+
if (this.disabled || this.getInterval() === 0) {
116133
return;
117134
}
118-
this.timer = setTimeout(() => {
119-
this.sendMetrics();
120-
}, this.metricsInterval + this.getAppliedJitter());
135+
this.timer = setTimeout(
136+
() => {
137+
this.sendMetrics();
138+
},
139+
this.getInterval(),
140+
);
121141

122142
if (process.env.NODE_ENV !== 'test' && typeof this.timer.unref === 'function') {
123143
this.timer.unref();
124144
}
125145
}
126146

127147
start(): void {
128-
if (typeof this.metricsInterval === 'number' && this.metricsInterval > 0) {
148+
if (this.metricsInterval > 0) {
129149
this.startTimer();
130150
this.registerInstance();
131151
}
@@ -170,6 +190,19 @@ export default class Metrics extends EventEmitter {
170190
return true;
171191
}
172192

193+
configurationError(url: string, statusCode: number) {
194+
this.emit(UnleashEvents.Warn, `${url} returning ${statusCode}, stopping metrics`);
195+
this.metricsInterval = 0;
196+
this.stop();
197+
}
198+
199+
backoff(url: string, statusCode: number): void {
200+
this.failures = Math.min(10, this.failures + 1);
201+
// eslint-disable-next-line max-len
202+
this.emit(UnleashEvents.Warn, `${url} returning ${statusCode}. Backing off to ${this.failures} times normal interval`);
203+
this.startTimer();
204+
}
205+
173206
async sendMetrics(): Promise<void> {
174207
if (this.disabled) {
175208
return;
@@ -194,16 +227,22 @@ export default class Metrics extends EventEmitter {
194227
timeout: this.timeout,
195228
httpOptions: this.httpOptions,
196229
});
197-
this.startTimer();
198-
if (res.status === 404) {
199-
this.emit(UnleashEvents.Warn, `${url} returning 404, stopping metrics`);
200-
this.stop();
201-
}
202230
if (!res.ok) {
231+
if (res.status === 404 || res.status === 403 || res.status == 401) {
232+
this.configurationError(url, res.status);
233+
} else if (
234+
res.status === 429 ||
235+
res.status === 500 ||
236+
res.status === 502 ||
237+
res.status === 503 ||
238+
res.status === 504
239+
) {
240+
this.backoff(url, res.status);
241+
}
203242
this.restoreBucket(payload.bucket);
204-
this.emit(UnleashEvents.Warn, `${url} returning ${res.status}`, await res.text());
205243
} else {
206244
this.emit(UnleashEvents.Sent, payload);
245+
this.reduceBackoff();
207246
}
208247
} catch (err) {
209248
this.restoreBucket(payload.bucket);
@@ -212,6 +251,11 @@ export default class Metrics extends EventEmitter {
212251
}
213252
}
214253

254+
reduceBackoff(): void {
255+
this.failures = Math.max(0, this.failures - 1);
256+
this.startTimer();
257+
}
258+
215259
assertBucket(name: string): void {
216260
if (this.disabled) {
217261
return;
@@ -243,7 +287,7 @@ export default class Metrics extends EventEmitter {
243287
}
244288

245289
private increaseCounter(name: string, enabled: boolean, inc = 1): void {
246-
if(inc === 0) {
290+
if (inc === 0) {
247291
return;
248292
}
249293
this.assertBucket(name);
@@ -252,8 +296,8 @@ export default class Metrics extends EventEmitter {
252296

253297
private increaseVariantCounter(name: string, variantName: string, inc = 1): void {
254298
this.assertBucket(name);
255-
if(this.bucket.toggles[name].variants[variantName]) {
256-
this.bucket.toggles[name].variants[variantName]+=inc
299+
if (this.bucket.toggles[name].variants[variantName]) {
300+
this.bucket.toggles[name].variants[variantName] += inc;
257301
} else {
258302
this.bucket.toggles[name].variants[variantName] = inc;
259303
}
@@ -276,7 +320,7 @@ export default class Metrics extends EventEmitter {
276320
}
277321

278322
createMetricsData(): MetricsData {
279-
const bucket = {...this.bucket, stop: new Date()};
323+
const bucket = { ...this.bucket, stop: new Date() };
280324
this.resetBucket();
281325
return {
282326
appName: this.appName,
@@ -286,20 +330,20 @@ export default class Metrics extends EventEmitter {
286330
}
287331

288332
private restoreBucket(bucket: Bucket): void {
289-
if(this.disabled) {
333+
if (this.disabled) {
290334
return;
291335
}
292336
this.bucket.start = bucket.start;
293337

294338
const { toggles } = bucket;
295-
Object.keys(toggles).forEach(toggleName => {
296-
const toggle = toggles[toggleName];
339+
Object.keys(toggles).forEach((toggleName) => {
340+
const toggle = toggles[toggleName];
297341
this.increaseCounter(toggleName, true, toggle.yes);
298342
this.increaseCounter(toggleName, false, toggle.no);
299343

300-
Object.keys(toggle.variants).forEach(variant => {
344+
Object.keys(toggle.variants).forEach((variant) => {
301345
this.increaseVariantCounter(toggleName, variant, toggle.variants[variant]);
302-
})
346+
});
303347
});
304348
}
305349

‎src/repository/index.ts

Lines changed: 88 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ export default class Repository extends EventEmitter implements EventEmitter {
5555

5656
private headers?: CustomHeaders;
5757

58+
private failures: number = 0;
59+
5860
private customHeadersFunction?: CustomHeadersFunction;
5961

6062
private timeout?: number;
@@ -241,11 +243,92 @@ Message: ${err.message}`,
241243
return obj;
242244
}
243245

246+
getFailures(): number {
247+
return this.failures;
248+
}
249+
250+
nextFetch(): number {
251+
return this.refreshInterval + this.failures * this.refreshInterval;
252+
}
253+
254+
private backoff(): number {
255+
this.failures = Math.min(this.failures + 1, 10);
256+
return this.nextFetch();
257+
}
258+
259+
private countSuccess(): number {
260+
this.failures = Math.max(this.failures - 1, 0);
261+
return this.nextFetch();
262+
}
263+
264+
// Emits correct error message based on what failed,
265+
// and returns 0 as the next fetch interval (stop polling)
266+
private configurationError(url: string, statusCode: number): number {
267+
this.failures += 1;
268+
if (statusCode === 404) {
269+
this.emit(
270+
UnleashEvents.Error,
271+
new Error(
272+
// eslint-disable-next-line max-len
273+
`${url} responded NOT_FOUND (404) which means your API url most likely needs correction. Stopping refresh of toggles`,
274+
),
275+
);
276+
} else if (statusCode === 401 || statusCode === 403) {
277+
this.emit(
278+
UnleashEvents.Error,
279+
new Error(
280+
// eslint-disable-next-line max-len
281+
`${url} responded ${statusCode} which means your API key is not allowed to connect. Stopping refresh of toggles`,
282+
),
283+
);
284+
}
285+
return 0;
286+
}
287+
288+
// We got a status code we know what to do with, so will log correct message
289+
// and return the new interval.
290+
private recoverableError(url: string, statusCode: number): number {
291+
let nextFetch = this.backoff();
292+
if (statusCode === 429) {
293+
this.emit(
294+
UnleashEvents.Warn,
295+
// eslint-disable-next-line max-len
296+
`${url} responded TOO_MANY_CONNECTIONS (429). Backing off`,
297+
);
298+
} else if (statusCode === 500 ||
299+
statusCode === 502 ||
300+
statusCode === 503 ||
301+
statusCode === 504) {
302+
this.emit(
303+
UnleashEvents.Warn,
304+
`${url} responded ${statusCode}. Backing off`,
305+
);
306+
}
307+
return nextFetch;
308+
}
309+
310+
private handleErrorCases(url: string, statusCode: number): number {
311+
if (statusCode === 401 || statusCode === 403 || statusCode === 404) {
312+
return this.configurationError(url, statusCode);
313+
} else if (
314+
statusCode === 429 ||
315+
statusCode === 500 ||
316+
statusCode === 502 ||
317+
statusCode === 503 ||
318+
statusCode === 504
319+
) {
320+
return this.recoverableError(url, statusCode);
321+
} else {
322+
const error = new Error(`Response was not statusCode 2XX, but was ${statusCode}`);
323+
this.emit(UnleashEvents.Error, error);
324+
return this.refreshInterval;
325+
}
326+
}
327+
244328
async fetch(): Promise<void> {
245329
if (this.stopped || !(this.refreshInterval > 0)) {
246330
return;
247331
}
248-
249332
let nextFetch = this.refreshInterval;
250333
try {
251334
let mergedTags;
@@ -257,7 +340,6 @@ Message: ${err.message}`,
257340
const headers = this.customHeadersFunction
258341
? await this.customHeadersFunction()
259342
: this.headers;
260-
261343
const res = await get({
262344
url,
263345
etag: this.etag,
@@ -268,14 +350,11 @@ Message: ${err.message}`,
268350
httpOptions: this.httpOptions,
269351
supportedSpecVersion: SUPPORTED_SPEC_VERSION,
270352
});
271-
272353
if (res.status === 304) {
273354
// No new data
274355
this.emit(UnleashEvents.Unchanged);
275-
} else if (!res.ok) {
276-
const error = new Error(`Response was not statusCode 2XX, but was ${res.status}`);
277-
this.emit(UnleashEvents.Error, error);
278-
} else {
356+
} else if (res.ok) {
357+
nextFetch = this.countSuccess();
279358
try {
280359
const data: ClientFeaturesResponse = await res.json();
281360
if (res.headers.get('etag') !== null) {
@@ -287,6 +366,8 @@ Message: ${err.message}`,
287366
} catch (err) {
288367
this.emit(UnleashEvents.Error, err);
289368
}
369+
} else {
370+
nextFetch = this.handleErrorCases(url, res.status);
290371
}
291372
} catch (err) {
292373
const e = err as { code: string };

0 commit comments

Comments
 (0)