Transaction Retry Implementation (#111)

* Adding retry logic. ToDo

* Added retry logic. Need to cleanup logs. First draft

* commented out erroring tx when retrying

* updated SQL for new columns updated retry logic

* added retry end-point with overrides gas values as input for retry

* updated processTransaction to remove retry code

* updated retry logic

* updated logs

* updated retry end-point to reset number of retries

* updated min block to wait before retry to 50 & Max gas values to 55gwei

* Added process exit if init scripts fail & added ability to create Database on psql

* added ensureDatabaseExists as a startup check

* Added wallet-type local to return when no ppk/gcp/aws is used

* updated import

* handled walletType for import. Local not supported for now

* added ssl rejectUnauthorized, from https://node-postgres.com/announcements#2020-02-25

* updates for ensureDatabaseExists

* hack. need to change this

Author: farhanW3

.env.example

@@ -68,4 +68,15 @@ BENCHMARK_POST_BODY='{
     "args": ["0x1946267d81Fb8aDeeEa28e6B98bcD446c8248473", 100000]
 }'
 BENCHMARK_CONCURRENCY=10
-BENCHMARK_REQUESTS=10
+BENCHMARK_REQUESTS=10
+
+# Retey Gas Max Values (All in wei)
+# Default Values
+# MAX_FEE_PER_GAS_FOR_RETRY=55000000000 (55 Gwei)
+# MAX_PRIORITY_FEE_PER_GAS_FOR_RETRY=55000000000 (55 Gwei)
+# MAX_RETRIES_FOR_TX=3
+# MAX_BLOCKS_ELAPSED_BEFORE_RETRY=50
+# RETRY_TX_CRON_SCHEDULE=*/30 * * * * *
+# RETRY_TX_ENABLED=true
+MAX_FEE_PER_GAS_FOR_RETRY=55000000000
+MAX_PRIORITY_FEE_PER_GAS_FOR_RETRY=55000000000

core/database/dbConnect.ts

@@ -1,82 +1,24 @@
-import { FastifyInstance, FastifyRequest } from "fastify";
 import pg, { Knex } from "knex";
 import { env } from "../env";
 
 // Defaults to postgres
 const dbClient = env.DATABASE_CLIENT;
 const connectionString = env.POSTGRES_CONNECTION_URL;
 
-const DATABASE_NAME =
-  new URL(connectionString).pathname.split("/")[1] || "postgres";
-
-export const connectToDB = async (
-  server: FastifyInstance | FastifyRequest,
+export const connectToDatabase = async (
+  databaseURL?: string,
 ): Promise<Knex> => {
-  // Creating KNEX Config
-
   let knexConfig: Knex.Config = {
     client: dbClient,
-    connection: connectionString,
+    connection: {
+      connectionString: databaseURL || connectionString,
+      ssl: {
+        rejectUnauthorized: false,
+      },
+    },
     acquireConnectionTimeout: 10000,
   };
 
-  // Set the appropriate databse client package
-  let dbClientPackage: any;
-  switch (dbClient) {
-    case "pg":
-      dbClientPackage = pg;
-      break;
-    default:
-      throw new Error(`Unsupported database client: ${dbClient}`);
-  }
-
-  let knex = dbClientPackage(knexConfig);
-
-  // Check if Database Exists & create if it doesn't
-  try {
-    let hasDatabase: any;
-    switch (dbClient) {
-      case "pg":
-        server.log.debug("checking if pg database exists");
-        hasDatabase = await knex.raw(
-          `SELECT 1 from pg_database WHERE datname = '${DATABASE_NAME}'`,
-        );
-        server.log.info(`CHECKING for Database ${DATABASE_NAME}...`);
-        if (!hasDatabase.rows.length) {
-          await knex.raw(`CREATE DATABASE ${DATABASE_NAME}`);
-        } else {
-          server.log.info(`Database ${DATABASE_NAME} already exists`);
-        }
-        break;
-      default:
-        throw new Error(
-          `Unsupported database client: ${dbClient}. Cannot create database ${DATABASE_NAME}`,
-        );
-    }
-  } catch (err) {
-    server.log.error(err);
-    throw new Error(`Error creating database ${DATABASE_NAME}`);
-  }
-
-  // Updating knex Config
-  knexConfig = {
-    client: dbClient,
-    connection: connectionString,
-  };
-
-  await knex.destroy();
-  // re-instantiate connection with new config
-  knex = dbClientPackage(knexConfig);
-
-  return knex;
-};
-
-export const connectWithDatabase = async (): Promise<Knex> => {
-  let knexConfig: Knex.Config = {
-    client: dbClient,
-    connection: connectionString,
-  };
-
   // Set the appropriate databse client package
   let dbClientPackage: typeof pg;
   switch (dbClient) {

core/database/dbOperation.ts

@@ -3,11 +3,11 @@ import { getSupportedChains } from "@thirdweb-dev/sdk";
 import { BigNumber } from "ethers";
 import { FastifyInstance } from "fastify";
 import { Knex } from "knex";
-import { getInstanceAdminWalletType, getWalletBackUpType } from "../helpers";
+import { getWalletType } from "../helpers";
 import { WalletData } from "../interfaces";
 import { getSDK } from "../sdk/sdk";
 import { getWalletNonce } from "../services/blockchain";
-import { connectWithDatabase } from "./dbConnect";
+import { connectToDatabase } from "./dbConnect";
 
 interface WalletExtraData {
   awsKmsKeyId?: string;
@@ -40,7 +40,7 @@ export const getWalletDetails = async (
     let passedAsParameter = true;
     if (!database) {
       passedAsParameter = false;
-      database = await connectWithDatabase();
+      database = await connectToDatabase();
     }
     const walletDetails = await database("wallets")
       .select("*")
@@ -103,9 +103,7 @@ export const addWalletDataWithSupportChainsNonceToDB = async (
         let lastUsedNonce = -1;
         let walletType = extraTableData?.walletType
           ? extraTableData?.walletType
-          : isWeb3APIInitWallet
-          ? getInstanceAdminWalletType()
-          : getWalletBackUpType();
+          : getWalletType();
         const sdk = await getSDK(slug, {
           walletAddress,
           walletType,

core/database/dbPrereqs.ts

@@ -2,20 +2,25 @@ import { promises as fs } from "fs";
 import { dirname } from "path";
 import { fileURLToPath } from "url";
 
-import { FastifyInstance } from "fastify";
+import { FastifyInstance, FastifyRequest } from "fastify";
 import { StatusCodes } from "http-status-codes";
 import { env } from "../env";
 import { createCustomError } from "../error/customError";
-import { connectToDB, connectWithDatabase } from "./dbConnect";
+import { connectToDatabase } from "./dbConnect";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
+const dbClient = env.DATABASE_CLIENT;
+const connectionString = env.POSTGRES_CONNECTION_URL;
+
+const DATABASE_NAME =
+  new URL(connectionString).pathname.split("/")[1] || "postgres";
 
 export const checkTablesExistence = async (
   server: FastifyInstance,
 ): Promise<void> => {
   try {
-    const knex = await connectToDB(server);
+    const knex = await connectToDatabase();
     // Check if the tables Exists
     const tablesList: string[] = env.DB_TABLES_LIST.split(",").map(function (
       item: any,
@@ -63,7 +68,7 @@ export const implementTriggerOnStartUp = async (
 ): Promise<void> => {
   try {
     // Connect to the DB
-    const knex = await connectWithDatabase();
+    const knex = await connectToDatabase();
 
     const triggersList: string[] = env.DB_TRIGGERS_LIST.split(",").map(
       function (item: any) {
@@ -113,3 +118,49 @@ export const implementTriggerOnStartUp = async (
     throw customError;
   }
 };
+
+export const ensureDatabaseExists = async (
+  server: FastifyInstance | FastifyRequest,
+): Promise<void> => {
+  try {
+    // Creating KNEX Config
+    let modifiedConnectionString = connectionString;
+    if (DATABASE_NAME !== "postgres") {
+      // This is required if the Database mentioned in the connection string is not postgres
+      // as we need to connect to the postgres database to create the user provied database
+      // and then connect to the user provided database
+      modifiedConnectionString = connectionString.replace(
+        `/${DATABASE_NAME}`,
+        "/postgres",
+      );
+    }
+
+    let knex = await connectToDatabase(modifiedConnectionString);
+
+    // Check if Database Exists & create if it doesn't
+    let hasDatabase: any;
+    switch (dbClient) {
+      case "pg":
+        server.log.debug("checking if pg database exists");
+        hasDatabase = await knex.raw(
+          `SELECT 1 from pg_database WHERE datname = '${DATABASE_NAME}'`,
+        );
+        server.log.info(`CHECKING for Database ${DATABASE_NAME}...`);
+        if (!hasDatabase.rows.length) {
+          await knex.raw(`CREATE DATABASE ${DATABASE_NAME}`);
+        } else {
+          server.log.info(`Database ${DATABASE_NAME} already exists`);
+        }
+        break;
+      default:
+        throw new Error(
+          `Unsupported database client: ${dbClient}. Cannot create database ${DATABASE_NAME}`,
+        );
+    }
+
+    await knex.destroy();
+  } catch (error) {
+    server.log.error(error);
+    throw new Error(`Error creating database ${DATABASE_NAME}`);
+  }
+};

core/database/sql-schemas/transactions.sql

@@ -26,7 +26,12 @@ CREATE TABLE IF NOT EXISTS transactions (
     "maxFeePerGas" VARCHAR(255),
     "txValue" VARCHAR(100),
     "txMinedTimestamp" TIMESTAMP,
-    "blockNumber" BIGINT
+    "blockNumber" BIGINT,
+    "numberOfRetries" INTEGER DEFAULT 0,
+    "overrideGasValuesForTx" BOOLEAN DEFAULT FALSE,
+    "txSubmittedAtBlockNumber" BIGINT,
+    "overrideMaxPriorityFeePerGas" VARCHAR(255),
+    "overrideMaxFeePerGas" VARCHAR(255)
 );
 
 ALTER TABLE transactions
@@ -46,4 +51,9 @@ ADD COLUMN IF NOT EXISTS "contractType" VARCHAR(255),
 ADD COLUMN IF NOT EXISTS "errorMessage" TEXT DEFAULT NULL,
 ADD COLUMN IF NOT EXISTS "txValue" VARCHAR(100) DEFAULT NULL,
 ADD COLUMN IF NOT EXISTS "txMinedTimestamp" TIMESTAMP,
-ADD COLUMN IF NOT EXISTS "blockNumber" BIGINT;
+ADD COLUMN IF NOT EXISTS "blockNumber" BIGINT,
+ADD COLUMN IF NOT EXISTS "numberOfRetries" INTEGER DEFAULT 0,
+ADD COLUMN IF NOT EXISTS "overrideGasValuesForTx" BOOLEAN DEFAULT FALSE,
+ADD COLUMN IF NOT EXISTS "txSubmittedAtBlockNumber" BIGINT,
+ADD COLUMN IF NOT EXISTS "overrideMaxPriorityFeePerGas" VARCHAR(255),
+ADD COLUMN IF NOT EXISTS "overrideMaxFeePerGas" VARCHAR(255);

core/env.ts

@@ -57,6 +57,13 @@ export const env = createEnv({
     GOOGLE_KMS_KEY_VERSION_ID: z.string().min(1).optional(),
     GOOGLE_APPLICATION_CREDENTIAL_EMAIL: z.string().min(1).optional(),
     GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY: z.string().min(1).optional(),
+    RETRY_TX_ENABLED: boolSchema("true"),
+    MAX_FEE_PER_GAS_FOR_RETRY: z.string().default("55000000000"),
+    MAX_PRIORITY_FEE_PER_GAS_FOR_RETRY: z.string().default("55000000000"),
+    MAX_RETRIES_FOR_TX: z.coerce.number().default(3),
+    RETRY_TX_CRON_SCHEDULE: z.string().default("*/30 * * * * *"),
+    MAX_BLOCKS_ELAPSED_BEFORE_RETRY: z.coerce.number().default(50),
+    MAX_WAIT_TIME_BEFORE_RETRY: z.coerce.number().default(600),
   },
   clientPrefix: "NEVER_USED",
   client: {},
@@ -94,6 +101,15 @@ export const env = createEnv({
       process.env.GOOGLE_APPLICATION_CREDENTIAL_EMAIL,
     GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY:
       process.env.GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY,
+    RETRY_TX_ENABLED: process.env.RETRY_TX_ENABLED,
+    MAX_FEE_PER_GAS_FOR_RETRY: process.env.MAX_FEE_PER_GAS_FOR_RETRY,
+    MAX_PRIORITY_FEE_PER_GAS_FOR_RETRY:
+      process.env.MAX_PRIORITY_FEE_PER_GAS_FOR_RETRY,
+    MAX_RETRIES_FOR_TX: process.env.MAX_RETRIES_FOR_TX,
+    RETRY_TX_CRON_SCHEDULE: process.env.RETRY_TX_CRON_SCHEDULE,
+    MAX_BLOCKS_ELAPSED_BEFORE_RETRY:
+      process.env.MAX_BLOCKS_ELAPSED_BEFORE_RETRY,
+    MAX_WAIT_TIME_BEFORE_RETRY: process.env.MAX_WAIT_TIME_BEFORE_RETRY,
   },
   onValidationError: (error: ZodError) => {
     console.error(

core/helpers/index.ts

@@ -18,7 +18,7 @@ export const isValidHttpUrl = (urlString: string): boolean => {
   return url.protocol === "http:" || url.protocol === "https:";
 };
 
-export const getInstanceAdminWalletType = (): string => {
+export const getWalletType = (): string => {
   if (WALLET_PRIVATE_KEY) {
     return "ppk";
   }
@@ -27,15 +27,6 @@ export const getInstanceAdminWalletType = (): string => {
     return "aws_kms";
   }
 
-  // ToDo GCP KMS
-  return "gcp_kms";
-};
-
-export const getWalletBackUpType = (): string => {
-  if (AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_REGION) {
-    return "aws_kms";
-  }
-
   if (
     env.GOOGLE_APPLICATION_CREDENTIAL_EMAIL &&
     env.GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY &&
@@ -44,9 +35,9 @@ export const getWalletBackUpType = (): string => {
     env.GOOGLE_KMS_KEY_VERSION_ID &&
     env.GOOGLE_KMS_LOCATION_ID
   ) {
-    return "gcp_kms";
+    return "aws_kms";
   }
 
   // ToDo GCP KMS
-  return "ppk";
+  return "local";
 };

package.json

@@ -14,7 +14,7 @@
     "dev:worker": "nodemon --watch 'worker/**/*.ts' --watch 'core/**/*.ts' --exec 'npx tsx ./worker/index.ts'",
     "dev:infra": "docker compose -f ./docker-compose-infra.yml up -d",
     "build": "yarn && rm -rf dist && tsc -p ./tsconfig.json --outDir dist",
-    "start": "yarn start:server & sleep 10 && yarn start:worker",
+    "start": "yarn start:server & sleep 20 && yarn start:worker",
     "start:server": "node --experimental-specifier-resolution=node ./dist/server/index.js",
     "start:worker": "node --experimental-specifier-resolution=node ./dist/worker/index.js",
     "start:docker": "docker compose build && docker compose --env-file ./.env up --remove-orphans",

server/api/index.ts

@@ -3,8 +3,10 @@ import { getContractExtensions } from "./contract/metadata/extensions";
 import { readContract } from "./contract/read/read";
 import { writeToContract } from "./contract/write/write";
 
+// Transactions
 import { getAllTx } from "./transaction/getAll";
 import { getAllDeployedContracts } from "./transaction/getAllDeployedContracts";
+import { retryTransaction } from "./transaction/retry";
 import { checkTxStatus } from "./transaction/status";
 
 // Extensions
@@ -77,6 +79,7 @@ export const apiRoutes = async (fastify: FastifyInstance) => {
   await fastify.register(checkTxStatus);
   await fastify.register(getAllTx);
   await fastify.register(getAllDeployedContracts);
+  await fastify.register(retryTransaction);
 
   // Extensions
   await fastify.register(erc20Routes);