unused nonces worker

Author: arcoraven

.eslintrc.json

@@ -13,6 +13,8 @@
     "@typescript-eslint/no-unused-vars": [
       "error",
       { "argsIgnorePattern": "^_" }
-    ]
+    ],
+    "require-await": "off",
+    "@typescript-eslint/require-await": "error"
   }
 }

package.json

@@ -47,7 +47,7 @@
     "@types/base-64": "^1.0.2",
     "base-64": "^1.0.0",
     "body-parser": "^1.20.2",
-    "bullmq": "^5.7.8",
+    "bullmq": "^5.10.3",
     "cookie": "^0.5.0",
     "cookie-parser": "^1.4.6",
     "copyfiles": "^2.4.1",

src/db/transactions/db.ts

@@ -182,7 +182,7 @@ export class TransactionDB {
    * @param to Date?
    * @returns string[] List of queueIds
    */
-  static purgeTransactions = async (args: { from?: Date; to?: Date }) => {
+  static pruneTransactions = async (args: { from?: Date; to?: Date }) => {
     const { from, to } = args;
     const min = from ? toSeconds(from) : 0;
     const max = to ? toSeconds(to) : "+inf";

src/db/transactions/queueTx.ts

@@ -2,6 +2,7 @@ import type { DeployTransaction, Transaction } from "@thirdweb-dev/sdk";
 import { ERC4337EthersSigner } from "@thirdweb-dev/wallets/dist/declarations/src/evm/connectors/smart-wallet/lib/erc4337-signer";
 import { Address } from "thirdweb";
 import type { ContractExtension } from "../../schema/extension";
+import { maybeBigInt } from "../../utils/primitiveTypes";
 import { insertTransaction } from "../../utils/transaction/insertTransaction";
 
 interface QueueTxParams {
@@ -65,13 +66,9 @@ export const queueTx = async ({
         signerAddress,
         accountAddress,
         target,
-        gas: txOverrides?.gas ? BigInt(txOverrides.gas) : undefined,
-        maxFeePerGas: txOverrides?.maxFeePerGas
-          ? BigInt(txOverrides?.maxFeePerGas)
-          : undefined,
-        maxPriorityFeePerGas: txOverrides?.maxPriorityFeePerGas
-          ? BigInt(txOverrides?.maxPriorityFeePerGas)
-          : undefined,
+        gas: maybeBigInt(txOverrides?.gas),
+        maxFeePerGas: maybeBigInt(txOverrides?.maxFeePerGas),
+        maxPriorityFeePerGas: maybeBigInt(txOverrides?.maxPriorityFeePerGas),
         from: signerAddress,
       },
       idempotencyKey,
@@ -96,13 +93,9 @@ export const queueTx = async ({
         deployedContractType,
         from: fromAddress.toLowerCase() as Address,
         to: toAddress?.toLowerCase() as Address,
-        gas: txOverrides?.gas ? BigInt(txOverrides.gas) : undefined,
-        maxFeePerGas: txOverrides?.maxFeePerGas
-          ? BigInt(txOverrides?.maxFeePerGas)
-          : undefined,
-        maxPriorityFeePerGas: txOverrides?.maxPriorityFeePerGas
-          ? BigInt(txOverrides?.maxPriorityFeePerGas)
-          : undefined,
+        gas: maybeBigInt(txOverrides?.gas),
+        maxFeePerGas: maybeBigInt(txOverrides?.maxFeePerGas),
+        maxPriorityFeePerGas: maybeBigInt(txOverrides?.maxPriorityFeePerGas),
       },
       idempotencyKey,
       shouldSimulate: simulateTx,

src/db/wallets/walletNonce.ts

@@ -3,32 +3,83 @@ import { getChain } from "../../utils/chain";
 import { redis } from "../../utils/redis/redis";
 import { thirdwebClient } from "../../utils/sdk";
 
-const nonceKey = (chainId: number, walletAddress: Address) =>
+/**
+ * !IMPORTANT
+ *
+ * The nonce stored is the LAST USED NONCE.
+ * To get the next available nonce, use `incrWalletNonce()`.
+ */
+
+const lastUsedNonceKey = (chainId: number, walletAddress: Address) =>
   `nonce:${chainId}:${walletAddress}`;
 
+const unusedNoncesKey = (chainId: number, walletAddress: Address) =>
+  `nonce-unused:${chainId}:${walletAddress}`;
+
 /**
- * Increment to get the next unused nonce.
- * If a nonce is acquired this way, it must be used for an onchain transaction
- * (or cancelled onchain).
+ * Acquire an unused nonce.
+ * This should be used to send an EOA transaction with this nonce.
  * @param chainId
  * @param walletAddress
  * @returns number - The next unused nonce value for this wallet.
  */
-export const incrWalletNonce = async (
+export const getAndUpdateNonce = async (
   chainId: number,
   walletAddress: Address,
 ) => {
-  const key = nonceKey(chainId, walletAddress);
-  let nonce = await redis.incr(key);
+  // Acquire an unused nonce, if any.
+  let nonce = await getAndUpdateLowestUnusedNonce(chainId, walletAddress);
+  if (nonce) {
+    return nonce;
+  }
+
+  // Else increment the last used nonce.
+  const key = lastUsedNonceKey(chainId, walletAddress);
+  nonce = await redis.incr(key);
   if (nonce === 1) {
     // If INCR returned 1, the nonce was not set.
     // This may be a newly imported wallet. Sync and return the onchain nonce.
-    nonce = await _syncWalletNonce(chainId, walletAddress);
+    return await _syncNonce(chainId, walletAddress);
   }
   return nonce;
 };
 
-const _syncWalletNonce = async (
+/**
+ * Adds an unused nonce that can be reused by a future transaction.
+ * This should be used if the current transaction that acquired this nonce is not valid.
+ * @param chainId
+ * @param walletAddress
+ * @param nonce
+ */
+export const addUnusedNonce = async (
+  chainId: number,
+  walletAddress: Address,
+  nonce: number,
+) => {
+  const key = unusedNoncesKey(chainId, walletAddress);
+  await redis.zadd(key, nonce, nonce);
+};
+
+/**
+ * Acquires the lowest unused nonce.
+ * @param chainId
+ * @param walletAddress
+ * @returns
+ */
+export const getAndUpdateLowestUnusedNonce = async (
+  chainId: number,
+  walletAddress: Address,
+) => {
+  const key = unusedNoncesKey(chainId, walletAddress);
+  const res = await redis.zpopmin(key);
+  if (res.length > 0) {
+    // res will be [value, score] where the score is the nonce.
+    return parseInt(res[1]);
+  }
+  return null;
+};
+
+const _syncNonce = async (
   chainId: number,
   walletAddress: Address,
 ): Promise<number> => {
@@ -42,7 +93,7 @@ const _syncWalletNonce = async (
     address: walletAddress,
   });
 
-  const key = nonceKey(chainId, walletAddress);
+  const key = lastUsedNonceKey(chainId, walletAddress);
   await redis.set(key, transactionCount);
   return transactionCount;
 };
@@ -55,19 +106,16 @@ const _syncWalletNonce = async (
  * @param walletAddress
  * @returns number - The last used nonce value for this wallet.
  */
-export const getWalletNonce = async (
-  chainId: number,
-  walletAddress: Address,
-) => {
-  const key = nonceKey(chainId, walletAddress);
+export const getNonce = async (chainId: number, walletAddress: Address) => {
+  const key = lastUsedNonceKey(chainId, walletAddress);
   const nonce = await redis.get(key);
   return nonce ? parseInt(nonce) : 0;
 };
 
 /**
  * Delete all wallet nonces. Useful when the get out of sync.
  */
-export const deleteWalletNonces = async () => {
+export const deleteAllNonces = async () => {
   const keys = await redis.keys("nonce:*");
   if (keys.length > 0) {
     await redis.del(keys);

src/server/routes/backend-wallet/getNonce.ts

@@ -2,7 +2,7 @@ import { Static, Type } from "@sinclair/typebox";
 import { FastifyInstance } from "fastify";
 import { StatusCodes } from "http-status-codes";
 import { Address } from "thirdweb";
-import { getWalletNonce } from "../../../db/wallets/walletNonce";
+import { getNonce } from "../../../db/wallets/walletNonce";
 import { standardResponseSchema } from "../../schemas/sharedApiSchemas";
 import { walletParamSchema } from "../../schemas/wallet";
 import { getChainIdFromChain } from "../../utils/chain";
@@ -43,7 +43,7 @@ export const getBackendWalletNonce = async (fastify: FastifyInstance) => {
     handler: async (req, reply) => {
       const { chain, walletAddress } = req.params;
       const chainId = await getChainIdFromChain(chain);
-      const nonce = await getWalletNonce(chainId, walletAddress as Address);
+      const nonce = await getNonce(chainId, walletAddress as Address);
 
       reply.status(StatusCodes.OK).send({
         result: {

src/server/routes/backend-wallet/resetNonces.ts

@@ -1,7 +1,7 @@
 import { Static, Type } from "@sinclair/typebox";
 import { FastifyInstance } from "fastify";
 import { StatusCodes } from "http-status-codes";
-import { deleteWalletNonces } from "../../../db/wallets/walletNonce";
+import { deleteAllNonces } from "../../../db/wallets/walletNonce";
 import { standardResponseSchema } from "../../schemas/sharedApiSchemas";
 
 const responseSchema = Type.Object({
@@ -34,7 +34,7 @@ export const resetBackendWalletNonces = async (fastify: FastifyInstance) => {
       },
     },
     handler: async (req, reply) => {
-      await deleteWalletNonces();
+      await deleteAllNonces();
 
       reply.status(StatusCodes.OK).send({
         result: {

src/server/routes/backend-wallet/sendTransaction.ts

@@ -2,6 +2,7 @@ import { Static, Type } from "@sinclair/typebox";
 import { FastifyInstance } from "fastify";
 import { StatusCodes } from "http-status-codes";
 import { Address, Hex } from "thirdweb";
+import { maybeBigInt } from "../../../utils/primitiveTypes";
 import { insertTransaction } from "../../../utils/transaction/insertTransaction";
 import {
   requestQuerystringSchema,
@@ -92,13 +93,11 @@ export async function sendTransaction(fastify: FastifyInstance) {
             data: data as Hex,
             value: BigInt(value),
 
-            gas: txOverrides?.gas ? BigInt(txOverrides.gas) : undefined,
-            maxFeePerGas: txOverrides?.maxFeePerGas
-              ? BigInt(txOverrides.maxFeePerGas)
-              : undefined,
-            maxPriorityFeePerGas: txOverrides?.maxPriorityFeePerGas
-              ? BigInt(txOverrides.maxPriorityFeePerGas)
-              : undefined,
+            gas: maybeBigInt(txOverrides?.gas),
+            maxFeePerGas: maybeBigInt(txOverrides?.maxFeePerGas),
+            maxPriorityFeePerGas: maybeBigInt(
+              txOverrides?.maxPriorityFeePerGas,
+            ),
           },
           shouldSimulate: simulateTx,
           idempotencyKey,

src/server/routes/backend-wallet/simulateTransaction.ts

@@ -3,7 +3,7 @@ import { randomUUID } from "crypto";
 import { FastifyInstance } from "fastify";
 import { StatusCodes } from "http-status-codes";
 import { Address, Hex } from "thirdweb";
-import { simulateQueuedTransaction } from "../../../utils/transaction/simulateTransaction";
+import { simulateQueuedTransaction } from "../../../utils/transaction/simulateQueuedTransaction";
 import { QueuedTransaction } from "../../../utils/transaction/types";
 import { createCustomError } from "../../middleware/error";
 import {
@@ -15,7 +15,10 @@ import { getChainIdFromChain } from "../../utils/chain";
 
 // INPUT
 const ParamsSchema = Type.Object({
-  chain: Type.String(),
+  chain: Type.String({
+    examples: ["80002"],
+    description: "Chain ID",
+  }),
 });
 
 const simulateRequestBodySchema = Type.Object({
@@ -25,7 +28,7 @@ const simulateRequestBodySchema = Type.Object({
   value: Type.Optional(
     Type.String({
       examples: ["0"],
-      description: "The amount of native currency",
+      description: "The amount of native currency in wei",
     }),
   ),
   // Decoded transaction args

src/server/routes/backend-wallet/transfer.ts

@@ -10,9 +10,9 @@ import {
   toWei,
 } from "thirdweb";
 import { transfer as transferERC20 } from "thirdweb/extensions/erc20";
-import { isContractDeployed } from "thirdweb/utils";
+import { isContractDeployed, resolvePromisedValue } from "thirdweb/utils";
 import { getChain } from "../../../utils/chain";
-import { resolvePromisedValue } from "../../../utils/resolvePromisedValue";
+import { maybeBigInt } from "../../../utils/primitiveTypes";
 import { thirdwebClient } from "../../../utils/sdk";
 import { insertTransaction } from "../../../utils/transaction/insertTransaction";
 import { InsertedTransaction } from "../../../utils/transaction/types";
@@ -84,13 +84,9 @@ export async function transfer(fastify: FastifyInstance) {
       const currencyAddress = _currencyAddress.toLowerCase();
       const chainId = await getChainIdFromChain(chain);
       const gasOverrides = {
-        gas: txOverrides?.gas ? BigInt(txOverrides.gas) : undefined,
-        maxFeePerGas: txOverrides?.maxFeePerGas
-          ? BigInt(txOverrides.maxFeePerGas)
-          : undefined,
-        maxPriorityFeePerGas: txOverrides?.maxPriorityFeePerGas
-          ? BigInt(txOverrides.maxPriorityFeePerGas)
-          : undefined,
+        gas: maybeBigInt(txOverrides?.gas),
+        maxFeePerGas: maybeBigInt(txOverrides?.maxFeePerGas),
+        maxPriorityFeePerGas: maybeBigInt(txOverrides?.maxPriorityFeePerGas),
       };
 
       let insertedTransaction: InsertedTransaction;