fix: lower Redis batch size (#669)

* fix: lower Redis batch size

* remove deprecated tag

Author: arcoraven

.vscode/settings.json

@@ -10,5 +10,8 @@
   "typescript.enablePromptUseWorkspaceTsdk": true,
   "[prisma]": {
     "editor.defaultFormatter": "Prisma.prisma"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "biomejs.biome"
   }
 }

src/db/transactions/db.ts

@@ -1,4 +1,5 @@
 import superjson from "superjson";
+import { env } from "../../utils/env";
 import { redis } from "../../utils/redis/redis";
 import { AnyTransaction } from "../../utils/transaction/types";
 
@@ -38,9 +39,6 @@ export class TransactionDB {
   private static cancelledTransactionsKey = `transaction:cancelled`;
   private static erroredTransactionsKey = `transaction:errored`;
 
-  // ioredis has limits over 100k+ (source: https://github.com/redis/ioredis/issues/801).
-  private static REDIS_BATCH_SIZE = 100_000;
-
   /**
    * Inserts or replaces a transaction details.
    * Also adds to the appropriate "status" sorted set.
@@ -113,9 +111,13 @@ export class TransactionDB {
     }
 
     const result: AnyTransaction[] = [];
-    for (let i = 0; i < queueIds.length; i += this.REDIS_BATCH_SIZE) {
+    for (
+      let i = 0;
+      i < queueIds.length;
+      i += env.__EXPERIMENTAL_REDIS_BATCH_SIZE
+    ) {
       const keys = queueIds
-        .slice(i, i + this.REDIS_BATCH_SIZE)
+        .slice(i, i + env.__EXPERIMENTAL_REDIS_BATCH_SIZE)
         .map(this.transactionDetailsKey);
       const vals = await redis.mget(...keys);
 
@@ -139,9 +141,13 @@ export class TransactionDB {
     }
 
     let numDeleted = 0;
-    for (let i = 0; i < queueIds.length; i += this.REDIS_BATCH_SIZE) {
+    for (
+      let i = 0;
+      i < queueIds.length;
+      i += env.__EXPERIMENTAL_REDIS_BATCH_SIZE
+    ) {
       const keys = queueIds
-        .slice(i, i + this.REDIS_BATCH_SIZE)
+        .slice(i, i + env.__EXPERIMENTAL_REDIS_BATCH_SIZE)
         .map(this.transactionDetailsKey);
       numDeleted += await redis.unlink(...keys);
     }
@@ -177,10 +183,10 @@ export class TransactionDB {
       status === "mined"
         ? this.minedTransactionsKey
         : status === "cancelled"
-        ? this.cancelledTransactionsKey
-        : status === "errored"
-        ? this.erroredTransactionsKey
-        : this.queuedTransactionsKey;
+          ? this.cancelledTransactionsKey
+          : status === "errored"
+            ? this.erroredTransactionsKey
+            : this.queuedTransactionsKey;
 
     const queueIds = await redis.zrevrange(key, start, end);
     const transactions = await this.bulkGet(queueIds);

src/utils/env.ts

@@ -90,6 +90,10 @@ export const env = createEnv({
     // Sets the max amount of memory Redis can use.
     // "0" means use all available memory.
     REDIS_MAXMEMORY: z.string().default("0"),
+    // Sets the max batch Redis will handle in batch operations like MGET and UNLINK.
+    // This will be removed if a consistent batch size works for all use cases.
+    // ioredis has issues with batches over 100k+ (source: https://github.com/redis/ioredis/issues/801).
+    __EXPERIMENTAL_REDIS_BATCH_SIZE: z.coerce.number().default(50_000),
     // Sets the number of recent transactions to store. Older transactions are pruned periodically.
     // In testing, 100k transactions consumes ~300mb memory.
     TRANSACTION_HISTORY_COUNT: z.coerce.number().default(100_000),
@@ -130,6 +134,8 @@ export const env = createEnv({
       process.env.CONFIRM_TRANSACTION_QUEUE_CONCURRENCY,
     ENGINE_MODE: process.env.ENGINE_MODE,
     REDIS_MAXMEMORY: process.env.REDIS_MAXMEMORY,
+    __EXPERIMENTAL_REDIS_BATCH_SIZE:
+      process.env.__EXPERIMENTAL_REDIS_BATCH_SIZE,
     TRANSACTION_HISTORY_COUNT: process.env.TRANSACTION_HISTORY_COUNT,
     GLOBAL_RATE_LIMIT_PER_MIN: process.env.GLOBAL_RATE_LIMIT_PER_MIN,
     DD_TRACER_ACTIVATED: process.env.DD_TRACER_ACTIVATED,

src/worker/tasks/pruneTransactionsWorker.ts

@@ -1,4 +1,4 @@
-import { Job, Processor, Worker } from "bullmq";
+import { Worker, type Job, type Processor } from "bullmq";
 import { TransactionDB } from "../../db/transactions/db";
 import { pruneNonceMaps } from "../../db/wallets/nonceMap";
 import { env } from "../../utils/env";