Skip to content

Commit 1ef1f17

Browse files
mergify[bot]mergify[bot]
authored
feat(comlink): Per-route rate limiters for candles, orders, fills, ... (backport #3199) (#3205)
Added dedicated RateLimiterRedis instances in src/caches/rate-limiters.ts for: /orders → ordersRateLimiter /fills → fillsRateLimiter /candles → candlesRateLimiter /sparklines → sparklinesRateLimiter /historical-pnl → historicalPnlRateLimiter /pnl → pnlRateLimiter /fundingPayments → fundingRateLimiter Each limiter uses its own configurable key prefix and rate parameters. Extended configSchema with new per-route rate-limit parameters: RATE_LIMIT_<ENDPOINT>_POINTS RATE_LIMIT_<ENDPOINT>_DURATION_SECONDS Each limiter is tested and verified to be configurable
1 parent eac01c9 commit 1ef1f17

28 files changed

+649
-124
lines changed

‎indexer/services/comlink/__tests__/lib/rate-limit.test.ts‎

Lines changed: 455 additions & 37 deletions
Large diffs are not rendered by default.

‎indexer/services/comlink/src/caches/rate-limiters.ts‎

Lines changed: 105 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,27 +11,114 @@ export const ratelimitRedis: {
1111
config.RATE_LIMIT_REDIS_URL, config.REDIS_RECONNECT_TIMEOUT_MS,
1212
);
1313

14+
export function getDefaultRateLimiter(): RateLimiterRedis {
15+
return new RateLimiterRedis({
16+
storeClient: ratelimitRedis.client,
17+
points: config.RATE_LIMIT_GET_POINTS,
18+
duration: config.RATE_LIMIT_GET_DURATION_SECONDS,
19+
keyPrefix: `${config.SERVICE_NAME}/get`,
20+
});
21+
}
22+
1423
// Generic rate limiter for all GET requests, limits per IP
15-
export const getReqRateLimiter: RateLimiterRedis = new RateLimiterRedis({
16-
storeClient: ratelimitRedis.client,
17-
points: config.RATE_LIMIT_GET_POINTS,
18-
duration: config.RATE_LIMIT_GET_DURATION_SECONDS,
19-
keyPrefix: `${config.SERVICE_NAME}/get`,
20-
});
24+
export const defaultRateLimiter: RateLimiterRedis = getDefaultRateLimiter();
2125

2226
// Rate-limiter for /screen endpoint querying a compliance provider, limits per IP
23-
export const screenProviderLimiter: RateLimiterRedis = new RateLimiterRedis({
24-
storeClient: ratelimitRedis.client,
25-
points: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_POINTS,
26-
duration: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_DURATION_SECONDS,
27-
keyPrefix: `${config.SERVICE_NAME}/screen_providers`,
28-
});
27+
export function getScreenProviderLimiter(): RateLimiterRedis {
28+
return new RateLimiterRedis({
29+
storeClient: ratelimitRedis.client,
30+
points: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_POINTS,
31+
duration: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_DURATION_SECONDS,
32+
keyPrefix: `${config.SERVICE_NAME}/screen_providers`,
33+
});
34+
}
35+
export const screenProviderLimiter: RateLimiterRedis = getScreenProviderLimiter();
2936

3037
// Rate-limiter for /screen endpoint querying a compliance provider, limits the total calls made
3138
// across all IPs
32-
export const screenProviderGlobalLimiter: RateLimiterRedis = new RateLimiterRedis({
33-
storeClient: ratelimitRedis.client,
34-
points: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_GLOBAL_POINTS,
35-
duration: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_GLOBAL_DURATION_SECONDS,
36-
keyPrefix: `${config.SERVICE_NAME}/screen_providers_global`,
37-
});
39+
export function getScreenProviderGlobalLimiter(): RateLimiterRedis {
40+
return new RateLimiterRedis({
41+
storeClient: ratelimitRedis.client,
42+
points: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_GLOBAL_POINTS,
43+
duration: config.RATE_LIMIT_SCREEN_QUERY_PROVIDER_GLOBAL_DURATION_SECONDS,
44+
keyPrefix: `${config.SERVICE_NAME}/screen_providers_global`,
45+
});
46+
}
47+
export const screenProviderGlobalLimiter: RateLimiterRedis = getScreenProviderGlobalLimiter();
48+
49+
// Rate-limiter for /orders endpoint, limits per IP
50+
export function getOrdersRateLimiter(): RateLimiterRedis {
51+
return new RateLimiterRedis({
52+
storeClient: ratelimitRedis.client,
53+
points: config.RATE_LIMIT_ORDERS_POINTS,
54+
duration: config.RATE_LIMIT_ORDERS_DURATION_SECONDS,
55+
keyPrefix: `${config.SERVICE_NAME}/orders`,
56+
});
57+
}
58+
export const ordersRateLimiter: RateLimiterRedis = getOrdersRateLimiter();
59+
60+
// Rate-limiter for /fills endpoint, limits per IP
61+
export function getFillsRateLimiter(): RateLimiterRedis {
62+
return new RateLimiterRedis({
63+
storeClient: ratelimitRedis.client,
64+
points: config.RATE_LIMIT_FILLS_POINTS,
65+
duration: config.RATE_LIMIT_FILLS_DURATION_SECONDS,
66+
keyPrefix: `${config.SERVICE_NAME}/fills`,
67+
});
68+
}
69+
export const fillsRateLimiter: RateLimiterRedis = getFillsRateLimiter();
70+
71+
// Rate-limiter for /candles endpoint, limits per IP
72+
export function getCandlesRateLimiter(): RateLimiterRedis {
73+
return new RateLimiterRedis({
74+
storeClient: ratelimitRedis.client,
75+
points: config.RATE_LIMIT_CANDLES_POINTS,
76+
duration: config.RATE_LIMIT_CANDLES_DURATION_SECONDS,
77+
keyPrefix: `${config.SERVICE_NAME}/candles`,
78+
});
79+
}
80+
export const candlesRateLimiter: RateLimiterRedis = getCandlesRateLimiter();
81+
82+
// Rate-limiter for /sparklines endpoint, limits per IP
83+
export function getSparklinesRateLimiter(): RateLimiterRedis {
84+
return new RateLimiterRedis({
85+
storeClient: ratelimitRedis.client,
86+
points: config.RATE_LIMIT_SPARKLINES_POINTS,
87+
duration: config.RATE_LIMIT_SPARKLINES_DURATION_SECONDS,
88+
keyPrefix: `${config.SERVICE_NAME}/sparklines`,
89+
});
90+
}
91+
export const sparklinesRateLimiter: RateLimiterRedis = getSparklinesRateLimiter();
92+
93+
// Rate-limiter for /historical-pnl endpoint, limits per IP
94+
export function getHistoricalPnlRateLimiter(): RateLimiterRedis {
95+
return new RateLimiterRedis({
96+
storeClient: ratelimitRedis.client,
97+
points: config.RATE_LIMIT_HISTORICAL_PNL_POINTS,
98+
duration: config.RATE_LIMIT_HISTORICAL_PNL_DURATION_SECONDS,
99+
keyPrefix: `${config.SERVICE_NAME}/historical_pnl`,
100+
});
101+
}
102+
export const historicalPnlRateLimiter: RateLimiterRedis = getHistoricalPnlRateLimiter();
103+
104+
// Rate-limiter for /pnl endpoint, limits per IP
105+
export function getPnlRateLimiter(): RateLimiterRedis {
106+
return new RateLimiterRedis({
107+
storeClient: ratelimitRedis.client,
108+
points: config.RATE_LIMIT_PNL_POINTS,
109+
duration: config.RATE_LIMIT_PNL_DURATION_SECONDS,
110+
keyPrefix: `${config.SERVICE_NAME}/pnl`,
111+
});
112+
}
113+
export const pnlRateLimiter: RateLimiterRedis = getPnlRateLimiter();
114+
115+
// Rate-limiter for /funding endpoint, limits per IP
116+
export function getFundingRateLimiter(): RateLimiterRedis {
117+
return new RateLimiterRedis({
118+
storeClient: ratelimitRedis.client,
119+
points: config.RATE_LIMIT_FUNDING_POINTS,
120+
duration: config.RATE_LIMIT_FUNDING_DURATION_SECONDS,
121+
keyPrefix: `${config.SERVICE_NAME}/funding`,
122+
});
123+
}
124+
export const fundingRateLimiter: RateLimiterRedis = getFundingRateLimiter();

‎indexer/services/comlink/src/config.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,23 @@ export const configSchema = {
5757
// Expose setting compliance status, only set to true in dev/staging.
5858
EXPOSE_SET_COMPLIANCE_ENDPOINT: parseBoolean({ default: false }),
5959

60+
// TODO review and finalize per-route rate limits
61+
// Rate limits for costly endpoints
62+
RATE_LIMIT_ORDERS_POINTS: parseInteger({ default: 100 }),
63+
RATE_LIMIT_ORDERS_DURATION_SECONDS: parseInteger({ default: 10 }),
64+
RATE_LIMIT_FILLS_POINTS: parseInteger({ default: 100 }),
65+
RATE_LIMIT_FILLS_DURATION_SECONDS: parseInteger({ default: 10 }),
66+
RATE_LIMIT_CANDLES_POINTS: parseInteger({ default: 1000 }),
67+
RATE_LIMIT_CANDLES_DURATION_SECONDS: parseInteger({ default: 10 }),
68+
RATE_LIMIT_SPARKLINES_POINTS: parseInteger({ default: 100 }),
69+
RATE_LIMIT_SPARKLINES_DURATION_SECONDS: parseInteger({ default: 10 }),
70+
RATE_LIMIT_HISTORICAL_PNL_POINTS: parseInteger({ default: 100 }),
71+
RATE_LIMIT_HISTORICAL_PNL_DURATION_SECONDS: parseInteger({ default: 10 }),
72+
RATE_LIMIT_PNL_POINTS: parseInteger({ default: 100 }),
73+
RATE_LIMIT_PNL_DURATION_SECONDS: parseInteger({ default: 10 }),
74+
RATE_LIMIT_FUNDING_POINTS: parseInteger({ default: 100 }),
75+
RATE_LIMIT_FUNDING_DURATION_SECONDS: parseInteger({ default: 10 }),
76+
6077
// Affiliates config
6178
VOLUME_ELIGIBILITY_THRESHOLD: parseInteger({ default: 10_000 }),
6279

‎indexer/services/comlink/src/controllers/api/v4/addresses-controller.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ import {
4747
Route,
4848
} from 'tsoa';
4949

50-
import { getReqRateLimiter } from '../../../caches/rate-limiters';
50+
import { defaultRateLimiter } from '../../../caches/rate-limiters';
5151
import config from '../../../config';
5252
import { AccountVerificationRequiredAction, validateSignature, validateSignatureKeplr } from '../../../helpers/compliance/compliance-utils';
5353
import { complianceAndGeoCheck } from '../../../lib/compliance-and-geo-check';
@@ -378,7 +378,7 @@ class AddressesController extends Controller {
378378

379379
router.get(
380380
'/:address',
381-
rateLimiterMiddleware(getReqRateLimiter),
381+
rateLimiterMiddleware(defaultRateLimiter),
382382
addressesCacheControlMiddleware,
383383
...CheckAddressSchema,
384384
handleValidationErrors,
@@ -418,7 +418,7 @@ router.get(
418418

419419
router.get(
420420
'/:address/subaccountNumber/:subaccountNumber',
421-
rateLimiterMiddleware(getReqRateLimiter),
421+
rateLimiterMiddleware(defaultRateLimiter),
422422
addressesCacheControlMiddleware,
423423
...CheckSubaccountSchema,
424424
handleValidationErrors,
@@ -463,7 +463,7 @@ router.get(
463463

464464
router.get(
465465
'/:address/parentSubaccountNumber/:parentSubaccountNumber',
466-
rateLimiterMiddleware(getReqRateLimiter),
466+
rateLimiterMiddleware(defaultRateLimiter),
467467
addressesCacheControlMiddleware,
468468
...CheckParentSubaccountSchema,
469469
handleValidationErrors,
@@ -563,7 +563,7 @@ router.post(
563563

564564
router.post(
565565
'/:address/testNotification',
566-
rateLimiterMiddleware(getReqRateLimiter),
566+
rateLimiterMiddleware(defaultRateLimiter),
567567
noCacheControlMiddleware,
568568
...CheckAddressSchema,
569569
handleValidationErrors,

‎indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import {
1616
Post,
1717
} from 'tsoa';
1818

19-
import { getReqRateLimiter } from '../../../caches/rate-limiters';
19+
import { defaultRateLimiter } from '../../../caches/rate-limiters';
2020
import config from '../../../config';
2121
import { AccountVerificationRequiredAction, validateSignature, validateSignatureKeplr } from '../../../helpers/compliance/compliance-utils';
2222
import { InvalidParamError, NotFoundError, UnexpectedServerError } from '../../../lib/errors';
@@ -282,7 +282,7 @@ class AffiliatesController extends Controller {
282282

283283
router.get(
284284
'/metadata',
285-
rateLimiterMiddleware(getReqRateLimiter),
285+
rateLimiterMiddleware(defaultRateLimiter),
286286
affiliatesMetadataCacheControlMiddleware,
287287
...checkSchema({
288288
address: {
@@ -322,7 +322,7 @@ router.get(
322322

323323
router.get(
324324
'/address',
325-
rateLimiterMiddleware(getReqRateLimiter),
325+
rateLimiterMiddleware(defaultRateLimiter),
326326
affiliatesCacheControlMiddleware,
327327
...checkSchema({
328328
referralCode: {
@@ -468,7 +468,7 @@ router.post(
468468

469469
router.get(
470470
'/snapshot',
471-
rateLimiterMiddleware(getReqRateLimiter),
471+
rateLimiterMiddleware(defaultRateLimiter),
472472
affiliatesCacheControlMiddleware,
473473
...checkSchema({
474474
addressFilter: {
@@ -554,7 +554,7 @@ router.get(
554554

555555
router.get(
556556
'/total_volume',
557-
rateLimiterMiddleware(getReqRateLimiter),
557+
rateLimiterMiddleware(defaultRateLimiter),
558558
affiliatesCacheControlMiddleware,
559559
...checkSchema({
560560
address: {

‎indexer/services/comlink/src/controllers/api/v4/asset-positions-controller.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import {
2323
Controller, Get, Query, Route,
2424
} from 'tsoa';
2525

26-
import { getReqRateLimiter } from '../../../caches/rate-limiters';
26+
import { defaultRateLimiter } from '../../../caches/rate-limiters';
2727
import config from '../../../config';
2828
import { complianceAndGeoCheck } from '../../../lib/compliance-and-geo-check';
2929
import { NotFoundError } from '../../../lib/errors';
@@ -281,7 +281,7 @@ async function adjustAssetPositionsWithFunding(
281281

282282
router.get(
283283
'/',
284-
rateLimiterMiddleware(getReqRateLimiter),
284+
rateLimiterMiddleware(defaultRateLimiter),
285285
assetPositionsCacheControlMiddleware,
286286
...CheckSubaccountSchema,
287287
handleValidationErrors,
@@ -324,7 +324,7 @@ router.get(
324324

325325
router.get(
326326
'/parentSubaccountNumber',
327-
rateLimiterMiddleware(getReqRateLimiter),
327+
rateLimiterMiddleware(defaultRateLimiter),
328328
assetPositionsCacheControlMiddleware,
329329
...CheckParentSubaccountSchema,
330330
handleValidationErrors,

‎indexer/services/comlink/src/controllers/api/v4/candles-controller.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
Controller, Get, Path, Query, Route,
99
} from 'tsoa';
1010

11-
import { getReqRateLimiter } from '../../../caches/rate-limiters';
11+
import { candlesRateLimiter } from '../../../caches/rate-limiters';
1212
import config from '../../../config';
1313
import { handleControllerError } from '../../../lib/helpers';
1414
import { rateLimiterMiddleware } from '../../../lib/rate-limit';
@@ -52,7 +52,7 @@ class CandleController extends Controller {
5252

5353
router.get(
5454
'/perpetualMarkets/:ticker',
55-
rateLimiterMiddleware(getReqRateLimiter),
55+
rateLimiterMiddleware(candlesRateLimiter),
5656
candlesCacheControlMiddleware,
5757
...CheckLimitSchema,
5858
...CheckTickerParamSchema,

‎indexer/services/comlink/src/controllers/api/v4/compliance-controller.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import {
1818
} from 'tsoa';
1919

2020
import {
21-
getReqRateLimiter,
21+
defaultRateLimiter,
2222
screenProviderGlobalLimiter,
2323
screenProviderLimiter,
2424
} from '../../../caches/rate-limiters';
@@ -120,7 +120,7 @@ export class ComplianceControllerHelper extends Controller {
120120

121121
router.get(
122122
'/',
123-
rateLimiterMiddleware(getReqRateLimiter),
123+
rateLimiterMiddleware(defaultRateLimiter),
124124
...checkSchema({
125125
address: {
126126
in: ['query'],

‎indexer/services/comlink/src/controllers/api/v4/compliance-v2-controller.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import {
2424
Controller, Get, Path, Route,
2525
} from 'tsoa';
2626

27-
import { getReqRateLimiter } from '../../../caches/rate-limiters';
27+
import { defaultRateLimiter } from '../../../caches/rate-limiters';
2828
import config from '../../../config';
2929
import { complianceProvider } from '../../../helpers/compliance/compliance-clients';
3030
import {
@@ -133,7 +133,7 @@ class ComplianceV2Controller extends Controller {
133133

134134
router.get(
135135
'/screen/:address',
136-
rateLimiterMiddleware(getReqRateLimiter),
136+
rateLimiterMiddleware(defaultRateLimiter),
137137
...CheckAddressSchema,
138138
handleValidationErrors,
139139
ExportResponseCodeStats({ controllerName }),

‎indexer/services/comlink/src/controllers/api/v4/fills-controller.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
Controller, Get, Query, Route,
2222
} from 'tsoa';
2323

24-
import { getReqRateLimiter } from '../../../caches/rate-limiters';
24+
import { fillsRateLimiter } from '../../../caches/rate-limiters';
2525
import config from '../../../config';
2626
import { complianceAndGeoCheck } from '../../../lib/compliance-and-geo-check';
2727
import { NotFoundError } from '../../../lib/errors';
@@ -183,7 +183,7 @@ class FillsController extends Controller {
183183

184184
router.get(
185185
'/',
186-
rateLimiterMiddleware(getReqRateLimiter),
186+
rateLimiterMiddleware(fillsRateLimiter),
187187
fillsCacheControlMiddleware,
188188
...CheckSubaccountSchema,
189189
...CheckLimitAndCreatedBeforeOrAtSchema,
@@ -265,7 +265,7 @@ router.get(
265265

266266
router.get(
267267
'/parentSubaccountNumber',
268-
rateLimiterMiddleware(getReqRateLimiter),
268+
rateLimiterMiddleware(fillsRateLimiter),
269269
fillsCacheControlMiddleware,
270270
...CheckParentSubaccountSchema,
271271
...CheckLimitAndCreatedBeforeOrAtSchema,

0 commit comments

Comments
 (0)