Skip to content

Commit eeee4d1

Browse files
d4mrarcoraven
d4mr
and
arcoraven
authored
Added prometheus metrics for response, queued, sent, mined, nonces (#647)
* Added prometheus metrics for response, queued, sent, mined * revert changes to queueTx.ts queueTx calls insertTransaction, don't duplicate the metric * feat: make metrics better * chore: Remove unused PROMETHEUS_PUSHGATEWAY_URL variable * feat: Update Prometheus metric name for backend wallet nonces * fix register import * fix inconsistencies * chore: serve metrics on different port and server * feat: added register to histograms, fixed URL bug * refactor: init metrics server next to engine-server * remove unnecessary logging * as const labels, rename label with duration, more buckets * smaller buckets for request duration --------- Co-authored-by: Phillip Ho <arcoraven@gmail.com>
1 parent 1eeaa5a commit eeee4d1

File tree

9 files changed

+393
-5
lines changed

9 files changed

+393
-5
lines changed

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
"node-cron": "^3.0.2",
6868
"pg": "^8.11.3",
6969
"prisma": "^5.14.0",
70+
"prom-client": "^15.1.3",
7071
"superjson": "^2.2.1",
7172
"thirdweb": "^5.45.1",
7273
"uuid": "^9.0.1",

‎src/server/index.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { URL } from "url";
66
import { clearCacheCron } from "../utils/cron/clearCacheCron";
77
import { env } from "../utils/env";
88
import { logger } from "../utils/logger";
9+
import { metricsServer } from "../utils/prometheus";
910
import { withServerUsageReporting } from "../utils/usage";
1011
import { updateTxListener } from "./listeners/updateTxListener";
1112
import { withAdminRoutes } from "./middleware/adminRoutes";
@@ -16,6 +17,7 @@ import { withErrorHandler } from "./middleware/error";
1617
import { withExpress } from "./middleware/express";
1718
import { withRequestLogs } from "./middleware/logs";
1819
import { withOpenApi } from "./middleware/open-api";
20+
import { withPrometheus } from "./middleware/prometheus";
1921
import { withRateLimit } from "./middleware/rateLimit";
2022
import { withWebSocket } from "./middleware/websocket";
2123
import { withRoutes } from "./routes";
@@ -62,6 +64,7 @@ export const initServer = async () => {
6264

6365
await withCors(server);
6466
await withRequestLogs(server);
67+
await withPrometheus(server);
6568
await withErrorHandler(server);
6669
await withEnforceEngineMode(server);
6770
await withRateLimit(server);
@@ -75,6 +78,15 @@ export const initServer = async () => {
7578

7679
await server.ready();
7780

81+
// if metrics are enabled, expose the metrics endpoint
82+
if (env.METRICS_ENABLED) {
83+
await metricsServer.ready();
84+
metricsServer.listen({
85+
host: env.HOST,
86+
port: env.METRICS_PORT,
87+
});
88+
}
89+
7890
server.listen(
7991
{
8092
host: env.HOST,

‎src/server/middleware/prometheus.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
2+
import { env } from "../../utils/env";
3+
import { recordMetrics } from "../../utils/prometheus";
4+
5+
export const withPrometheus = async (server: FastifyInstance) => {
6+
if (!env.METRICS_ENABLED) {
7+
return;
8+
}
9+
10+
server.addHook(
11+
"onResponse",
12+
(req: FastifyRequest, res: FastifyReply, done) => {
13+
const { method } = req;
14+
const url = req.routeOptions.url;
15+
const { statusCode } = res;
16+
const duration = res.elapsedTime; // Use Fastify's built-in timing
17+
18+
// Record metrics
19+
recordMetrics({
20+
event: "response_sent",
21+
params: {
22+
endpoint: url ?? "",
23+
statusCode: statusCode.toString(),
24+
duration: duration,
25+
method: method,
26+
},
27+
});
28+
29+
done();
30+
},
31+
);
32+
};

‎src/utils/env.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ export const env = createEnv({
8080
GLOBAL_RATE_LIMIT_PER_MIN: z.coerce.number().default(400 * 60),
8181
DD_TRACER_ACTIVATED: z.coerce.boolean().default(false),
8282

83+
// Prometheus
84+
METRICS_PORT: z.coerce.number().default(4001),
85+
METRICS_ENABLED: boolSchema("false"),
86+
8387
/**
8488
* Limits
8589
*/
@@ -132,6 +136,8 @@ export const env = createEnv({
132136
QUEUE_COMPLETE_HISTORY_COUNT: process.env.QUEUE_COMPLETE_HISTORY_COUNT,
133137
QUEUE_FAIL_HISTORY_COUNT: process.env.QUEUE_FAIL_HISTORY_COUNT,
134138
NONCE_MAP_COUNT: process.env.NONCE_MAP_COUNT,
139+
METRICS_PORT: process.env.METRICS_PORT,
140+
METRICS_ENABLED: process.env.METRICS_ENABLED,
135141
},
136142
onValidationError: (error: ZodError) => {
137143
console.error(

‎src/utils/prometheus.ts

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
import fastify from "fastify";
2+
import { Counter, Gauge, Histogram, Registry } from "prom-client";
3+
import { getUsedBackendWallets, inspectNonce } from "../db/wallets/walletNonce";
4+
import { getLastUsedOnchainNonce } from "../server/routes/admin/nonces";
5+
6+
const nonceMetrics = new Gauge({
7+
name: "engine_nonces",
8+
help: "Current nonce values and health for backend wallets",
9+
labelNames: ["wallet_address", "chain_id", "nonce_type"] as const,
10+
async collect() {
11+
const allWallets = await getUsedBackendWallets();
12+
13+
for (const { chainId, walletAddress } of allWallets) {
14+
const [onchainNonce, engineNonce] = await Promise.all([
15+
getLastUsedOnchainNonce(chainId, walletAddress),
16+
inspectNonce(chainId, walletAddress),
17+
]);
18+
19+
this.set(
20+
{
21+
wallet_address: walletAddress,
22+
chain_id: chainId.toString(),
23+
nonce_type: "onchain",
24+
},
25+
onchainNonce,
26+
);
27+
this.set(
28+
{
29+
wallet_address: walletAddress,
30+
chain_id: chainId.toString(),
31+
nonce_type: "engine",
32+
},
33+
engineNonce,
34+
);
35+
}
36+
},
37+
});
38+
39+
export const enginePromRegister = new Registry();
40+
41+
enginePromRegister.registerMetric(nonceMetrics);
42+
43+
// Define strict types for each event
44+
type ResponseSentParams = {
45+
endpoint: string;
46+
statusCode: string;
47+
method: string;
48+
duration: number;
49+
};
50+
51+
type TransactionQueuedParams = {
52+
chainId: string;
53+
walletAddress: string;
54+
};
55+
56+
type TransactionSentParams = TransactionQueuedParams & {
57+
success: boolean;
58+
durationSeconds: number;
59+
};
60+
61+
type TransactionMinedParams = TransactionQueuedParams & {
62+
queuedToMinedDurationSeconds: number;
63+
durationSeconds: number;
64+
};
65+
66+
// Union type for all possible event parameters
67+
type MetricParams =
68+
| { event: "response_sent"; params: ResponseSentParams }
69+
| { event: "transaction_queued"; params: TransactionQueuedParams }
70+
| { event: "transaction_sent"; params: TransactionSentParams }
71+
| { event: "transaction_mined"; params: TransactionMinedParams };
72+
73+
// Request metrics
74+
const requestsTotal = new Counter({
75+
name: "engine_requests_total",
76+
help: "Total number of completed requests",
77+
labelNames: ["endpoint", "status_code", "method"],
78+
registers: [enginePromRegister],
79+
});
80+
81+
const requestDuration = new Histogram({
82+
name: "engine_response_time_ms",
83+
help: "Response time in milliseconds",
84+
labelNames: ["endpoint", "status_code", "method"],
85+
buckets: [1, 10, 50, 100, 300, 500, 700, 1000, 3000, 5000, 7000, 10_000],
86+
registers: [enginePromRegister],
87+
});
88+
89+
// Transaction metrics
90+
const transactionsQueued = new Gauge({
91+
name: "engine_transactions_queued",
92+
help: "Number of transactions currently in queue",
93+
labelNames: ["chain_id", "wallet_address"] as const,
94+
registers: [enginePromRegister],
95+
});
96+
97+
const transactionsQueuedTotal = new Counter({
98+
name: "engine_transactions_queued_total",
99+
help: "Total number of transactions queued",
100+
labelNames: ["chain_id", "wallet_address"] as const,
101+
registers: [enginePromRegister],
102+
});
103+
104+
const transactionsSentTotal = new Counter({
105+
name: "engine_transactions_sent_total",
106+
help: "Total number of transactions sent",
107+
labelNames: ["chain_id", "wallet_address", "is_success"] as const,
108+
registers: [enginePromRegister],
109+
});
110+
111+
const transactionsMinedTotal = new Counter({
112+
name: "engine_transactions_mined_total",
113+
help: "Total number of transactions mined",
114+
labelNames: ["chain_id", "wallet_address"] as const,
115+
registers: [enginePromRegister],
116+
});
117+
118+
// Transaction duration histograms
119+
const queuedToSentDuration = new Histogram({
120+
name: "engine_transaction_queued_to_sent_seconds",
121+
help: "Duration from queued to sent in seconds",
122+
labelNames: ["chain_id", "wallet_address"] as const,
123+
buckets: [
124+
0.05, // 50ms
125+
0.1, // 100ms
126+
0.5, // 500ms
127+
1, // 1s
128+
5, // 5s
129+
15, // 15s
130+
30, // 30s
131+
60, // 1m
132+
300, // 5m
133+
600, // 10m
134+
1800, // 30m
135+
3600, // 1h
136+
7200, // 2h
137+
21600, // 6h
138+
43200, // 12h
139+
],
140+
registers: [enginePromRegister],
141+
});
142+
143+
const sentToMinedDuration = new Histogram({
144+
name: "engine_transaction_sent_to_mined_seconds",
145+
help: "Duration from sent to mined in seconds",
146+
labelNames: ["chain_id", "wallet_address"] as const,
147+
buckets: [
148+
0.2, // 200ms
149+
0.5, // 500ms
150+
1, // 1s
151+
5, // 5s
152+
10, // 10s
153+
30, // 30s
154+
60, // 1m
155+
120, // 2m
156+
300, // 5m
157+
600, // 10m
158+
900, // 15m
159+
1200, // 20m
160+
1800, // 30m
161+
],
162+
registers: [enginePromRegister],
163+
});
164+
165+
const queuedToMinedDuration = new Histogram({
166+
name: "engine_transaction_queued_to_mined_seconds",
167+
help: "Duration from queued to mined in seconds",
168+
labelNames: ["chain_id", "wallet_address"] as const,
169+
buckets: [
170+
0.2, // 200ms
171+
0.5, // 500ms
172+
1, // 1s
173+
5, // 5s
174+
15, // 15s
175+
30, // 30s
176+
60, // 1m
177+
300, // 5m
178+
600, // 10m
179+
1800, // 30m
180+
3600, // 1h
181+
7200, // 2h
182+
21600, // 6h
183+
43200, // 12h
184+
],
185+
registers: [enginePromRegister],
186+
});
187+
188+
// Function to record metrics
189+
export function recordMetrics(eventData: MetricParams): void {
190+
const { event, params } = eventData;
191+
switch (event) {
192+
case "response_sent": {
193+
const labels = {
194+
endpoint: params.endpoint,
195+
status_code: params.statusCode,
196+
};
197+
requestsTotal.inc(labels);
198+
requestDuration.observe(labels, params.duration);
199+
break;
200+
}
201+
202+
case "transaction_queued": {
203+
const labels = {
204+
chain_id: params.chainId,
205+
wallet_address: params.walletAddress,
206+
};
207+
transactionsQueued.inc(labels);
208+
transactionsQueuedTotal.inc(labels);
209+
break;
210+
}
211+
212+
case "transaction_sent": {
213+
const labels = {
214+
chain_id: params.chainId,
215+
wallet_address: params.walletAddress,
216+
};
217+
transactionsQueued.dec(labels);
218+
transactionsSentTotal.inc({ ...labels, is_success: `${params.success}` });
219+
queuedToSentDuration.observe(labels, params.durationSeconds);
220+
break;
221+
}
222+
223+
case "transaction_mined": {
224+
const labels = {
225+
chain_id: params.chainId,
226+
wallet_address: params.walletAddress,
227+
};
228+
transactionsMinedTotal.inc(labels);
229+
sentToMinedDuration.observe(labels, params.durationSeconds);
230+
queuedToMinedDuration.observe(
231+
labels,
232+
params.queuedToMinedDurationSeconds,
233+
);
234+
break;
235+
}
236+
}
237+
}
238+
// Expose metrics endpoint
239+
240+
export const metricsServer = fastify({
241+
disableRequestLogging: true,
242+
});
243+
244+
metricsServer.get("/metrics", async (_request, reply) => {
245+
reply.header("Content-Type", enginePromRegister.contentType);
246+
return enginePromRegister.metrics();
247+
});

‎src/utils/transaction/insertTransaction.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { TransactionDB } from "../../db/transactions/db";
33
import { createCustomError } from "../../server/middleware/error";
44
import { SendTransactionQueue } from "../../worker/queues/sendTransactionQueue";
55
import { getChecksumAddress } from "../primitiveTypes";
6+
import { recordMetrics } from "../prometheus";
67
import { reportUsage } from "../usage";
78
import { doSimulateTransaction } from "./simulateQueuedTransaction";
89
import { InsertedTransaction, QueuedTransaction } from "./types";
@@ -70,5 +71,13 @@ export const insertTransaction = async (
7071
});
7172
reportUsage([{ action: "queue_tx", input: queuedTransaction }]);
7273

74+
recordMetrics({
75+
event: "transaction_queued",
76+
params: {
77+
chainId: queuedTransaction.chainId.toString(),
78+
walletAddress: queuedTransaction.from,
79+
},
80+
});
81+
7382
return queueId;
7483
};

0 commit comments

Comments
 (0)