Create EOA + Sign & Send Tx (#103)

* updated code to use from, AWS KMS EOA wallets

* updated end-points to include web3api overrides & creation of eoa on AWS, wallet end-points

* fixed e2e

* fix for value field, e2e fixed too

* removed comment

* using ethers gcp signer to get walletAddress. But having issues, so commented

Author: farhanW3

.env.benchmark

@@ -0,0 +1,13 @@
+THIRDWEB_SDK_SECRET_KEY=klRsqmatrdlEpik_pHKgYy_q2YzGe3bTewO1VC26eY_H184Kc7xOVqKVj0mHwOOW2AOx2N-a3GqLCQ7Z9s9-sw
+
+# benchmark vars
+NODE_ENV=benchmark
+BENCHMARK_HOST='http://0.0.0.0:3005'
+BENCHMARK_URL_PATH='/contract/mumbai/0x365b83D67D5539C6583b9c0266A548926Bf216F4/write'
+BENCHMARK_POST_BODY='{
+    "function_name": "transfer",
+    "args": ["0x1946267d81Fb8aDeeEa28e6B98bcD446c8248473", 100000]
+}'
+# Total request has to be more than the total concurrency
+BENCHMARK_CONCURRENCY=10
+BENCHMARK_REQUESTS=10

.gitignore

@@ -135,4 +135,8 @@ swagger.yml
 worker/distWorker/
 .DS_Store
 post_data.json
-benchmark.local.sh
+benchmark.local.sh
+
+# GCP Service Account JSON
+thrdweb-biyvmatstwcbrb08y692g-41baca7a34a1.json
+thrdweb-biyvmatstwcbrb08y692g-7762eca4411b.json

core/database/dbOperation.ts

@@ -1,12 +1,167 @@
+import { getChainBySlug } from "@thirdweb-dev/chains";
+import { getSupportedChains } from "@thirdweb-dev/sdk";
+import { BigNumber } from "ethers";
+import { FastifyInstance } from "fastify";
 import { Knex } from "knex";
+import { getInstanceAdminWalletType, getWalletBackUpType } from "../helpers";
 import { WalletData } from "../interfaces";
+import { getSDK } from "../sdk/sdk";
+import { getWalletNonce } from "../services/blockchain";
+import { connectWithDatabase } from "./dbConnect";
+
+interface WalletExtraData {
+  awsKmsKeyId?: string;
+  awsKmsArn?: string;
+  walletType?: string;
+  gcpKmsKeyId?: string;
+  gcpKmsKeyRingId?: string;
+}
 
 export const insertIntoWallets = async (
   walletData: WalletData,
   database: Knex,
-): Promise<void> => {
-  await database("wallets")
+): Promise<number[]> => {
+  return database("wallets")
     .insert(walletData)
     .onConflict(["walletAddress", "chainId"])
     .merge();
 };
+
+export const getWalletDetails = async (
+  walletAddress: string,
+  chainId: string,
+  database?: Knex,
+): Promise<any> => {
+  try {
+    let passedAsParameter = true;
+    if (!database) {
+      passedAsParameter = false;
+      database = await connectWithDatabase();
+    }
+    const walletDetails = await database("wallets")
+      .select("*")
+      .where({ walletAddress: walletAddress.toLowerCase(), chainId })
+      .orWhere({ walletAddress: walletAddress.toLowerCase(), slug: chainId })
+      .first();
+
+    if (!passedAsParameter) {
+      await database.destroy();
+    }
+    return walletDetails;
+  } catch (error) {
+    throw error;
+  }
+};
+
+export const addWalletToDB = async (
+  chainId: string,
+  walletAddress: string,
+  slug: string,
+  dbInstance: Knex,
+): Promise<void> => {
+  try {
+    const sdk = await getSDK(chainId);
+    const walletNonce = await getWalletNonce(
+      walletAddress.toLowerCase(),
+      sdk.getProvider(),
+    );
+
+    const walletData = {
+      walletAddress: walletAddress.toLowerCase(),
+      chainId: chainId.toLowerCase(),
+      blockchainNonce: BigNumber.from(walletNonce ?? 0).toNumber(),
+      lastSyncedTimestamp: new Date(),
+      lastUsedNonce: -1,
+      walletType: slug,
+    };
+
+    await insertIntoWallets(walletData, dbInstance);
+  } catch (error) {
+    throw error;
+  }
+};
+
+export const addWalletDataWithSupportChainsNonceToDB = async (
+  server: FastifyInstance,
+  dbInstance: Knex,
+  isWeb3APIInitWallet?: boolean,
+  walletAddress?: string,
+  extraTableData?: WalletExtraData,
+): Promise<void> => {
+  try {
+    server.log.info("Setting up wallet Table");
+    const supportedChains = await getSupportedChains();
+    const promises = supportedChains.map(async (chain) => {
+      try {
+        const { slug } = chain;
+        let lastUsedNonce = -1;
+        let walletType = isWeb3APIInitWallet
+          ? getInstanceAdminWalletType()
+          : getWalletBackUpType();
+        const sdk = await getSDK(slug, {
+          walletAddress,
+          walletType,
+          awsKmsKeyId: extraTableData?.awsKmsKeyId,
+        });
+        walletAddress =
+          (await sdk.getSigner()?.getAddress())?.toLowerCase() ?? "";
+        server.log.debug(`Setting up wallet for chain ${slug}`);
+        if (walletAddress.length === 0) {
+          // server.log.warn(`Wallet address not found for chain ${slug}.`);
+          throw new Error(`Wallet address not found for chain ${slug}.`);
+        }
+        const walletBlockchainNonce = await getWalletNonce(
+          walletAddress,
+          sdk.getProvider(),
+        );
+        const walletDataInDB = await getWalletDetails(
+          walletAddress,
+          BigNumber.from(chain.chainId).toString(),
+          dbInstance,
+        );
+
+        if (walletDataInDB) {
+          lastUsedNonce = walletDataInDB.lastUsedNonce;
+        }
+
+        // lastUsedNonce should be set to -1 if blockchainNonce is 0
+        if (
+          BigNumber.from(walletBlockchainNonce).eq(BigNumber.from(0)) &&
+          BigNumber.from(lastUsedNonce).eq(BigNumber.from(0))
+        ) {
+          lastUsedNonce = -1;
+        }
+
+        if (
+          extraTableData?.walletType &&
+          extraTableData?.walletType.length > 0
+        ) {
+          walletType = extraTableData.walletType;
+          delete extraTableData.walletType;
+        }
+
+        const walletData = {
+          ...extraTableData,
+          walletAddress: walletAddress.toLowerCase(),
+          chainId: getChainBySlug(slug).chainId.toString(),
+          blockchainNonce: BigNumber.from(
+            walletBlockchainNonce ?? 0,
+          ).toNumber(),
+          lastSyncedTimestamp: new Date(),
+          lastUsedNonce,
+          slug,
+          walletType,
+        };
+        return insertIntoWallets(walletData, dbInstance);
+      } catch (error) {
+        server.log.error((error as any).message);
+        return Promise.resolve();
+      }
+    });
+
+    await Promise.all(promises);
+    server.log.info(`Wallet Table setup completed`);
+  } catch (error) {
+    throw error;
+  }
+};

core/database/dbPrereqs.ts

@@ -48,6 +48,7 @@ export const checkTablesExistence = async (
     // Disconnect from DB
     await knex.destroy();
   } catch (error: any) {
+    server.log.error(error);
     const customError = createCustomError(
       "Error while executing Table SQLs on startup",
       StatusCodes.INTERNAL_SERVER_ERROR,

core/database/sql-schemas/transactions.sql

@@ -23,7 +23,8 @@ CREATE TABLE IF NOT EXISTS transactions (
     "gasPrice" VARCHAR(255),
     "gasLimit" VARCHAR(255),
     "maxPriorityFeePerGas" VARCHAR(255),
-    "maxFeePerGas" VARCHAR(255)
+    "maxFeePerGas" VARCHAR(255),
+    "txValue" VARCHAR(100)
 );
 
 ALTER TABLE transactions
@@ -41,5 +42,6 @@ ALTER COLUMN "txType" TYPE VARCHAR(255),
 ADD COLUMN IF NOT EXISTS "deployedContractAddress" VARCHAR(255),
 ADD COLUMN IF NOT EXISTS "contractType" VARCHAR(255),
 ADD COLUMN IF NOT EXISTS "errorMessage" TEXT DEFAULT NULL,
+ADD COLUMN IF NOT EXISTS "txValue" VARCHAR(100) DEFAULT NULL,
 ADD COLUMN IF NOT EXISTS "txMinedTimestamp" TIMESTAMP,
-ADD COLUMN IF NOT EXISTS "blockNumber" BIGINT;
+ADD COLUMN IF NOT EXISTS "blockNumber" BIGINT;

core/database/sql-schemas/wallets.sql

@@ -2,14 +2,27 @@ CREATE TABLE IF NOT EXISTS wallets (
     "walletAddress" VARCHAR(255) NOT NULL,
     "chainId" VARCHAR(255) NOT NULL,
     "walletType" VARCHAR(255) NOT NULL,
+    "slug" VARCHAR(255),
     "blockchainNonce" BIGINT NOT NULL,
     "lastSyncedTimestamp" TIMESTAMP,
     "lastUsedNonce" BIGINT NOT NULL,
+    "awsKmsKeyId" VARCHAR(255),
+    "awsKmsArn" VARCHAR(255),
+    -- "gcpKmsKeyRingId" VARCHAR(50),
+    -- "gcpKmsKeyId" VARCHAR(50),
+    -- "gcpKmsKeyVersion" VARCHAR(20),
     PRIMARY KEY ("walletAddress", "chainId")
 );
 
 ALTER TABLE wallets
 ALTER COLUMN "chainId" TYPE VARCHAR(255),
 ALTER COLUMN "walletType" TYPE VARCHAR(255),
 ALTER COLUMN "blockchainNonce" TYPE BIGINT,
-ALTER COLUMN "lastUsedNonce" TYPE BIGINT;
+ALTER COLUMN "lastUsedNonce" TYPE BIGINT,
+ADD COLUMN IF NOT EXISTS "awsKmsKeyId" VARCHAR(255),
+ADD COLUMN IF NOT EXISTS "awsKmsArn" VARCHAR(255),
+ADD COLUMN IF NOT EXISTS "slug" VARCHAR(255),
+DROP COLUMN IF EXISTS "chainName";
+-- ADD COLUMN IF NOT EXISTS "gcpKmsKeyRingId" VARCHAR(50),
+-- ADD COLUMN IF NOT EXISTS "gcpKmsKeyId" VARCHAR(50),
+-- ADD COLUMN IF NOT EXISTS "gcpKmsKeyVersion" VARCHAR(20);

core/env.ts

@@ -29,12 +29,12 @@ export const env = createEnv({
         WALLET_PRIVATE_KEY: z.string().min(1),
       }),
       z.object({
-        AWS_ACCESS_KEY_ID: z.string().min(1),
-        AWS_SECRET_ACCESS_KEY: z.string().min(1),
         AWS_KMS_KEY_ID: z.string().min(1),
-        AWS_REGION: z.string().min(1),
       }),
     ]),
+    AWS_ACCESS_KEY_ID: z.string().min(1).optional(),
+    AWS_SECRET_ACCESS_KEY: z.string().min(1).optional(),
+    AWS_REGION: z.string().min(1).optional(),
     THIRDWEB_SDK_SECRET_KEY: z.string().min(1),
     THIRDWEB_API_ORIGIN: z.string().default("http://api.thirdweb.com"),
     POSTGRES_HOST: z.string().default("localhost"),
@@ -56,8 +56,13 @@ export const env = createEnv({
     CHAIN_OVERRIDES: z.string().default(""),
     ACCESS_CONTROL_ALLOW_ORIGIN: z.string().default("*"),
     MINED_TX_CRON_ENABLED: boolSchema("true"),
-    MINED_TX_CRON_SCHEDULE: z.string().default("*/30 * * * * *"),
+    MINED_TX_CRON_SCHEDULE: z.string().default("*/5 * * * * *"),
     MIN_TX_TO_CHECK_FOR_MINED_STATUS: z.coerce.number().default(50),
+    GCP_PROJECT_ID: z.string().min(1).optional(),
+    GCP_KEY_RING_ID: z.string().min(1).optional(),
+    GCP_LOCATION_ID: z.string().min(1).optional(),
+    GOOGLE_APPLICATION_CREDENTIAL_EMAIL: z.string().min(1).optional(),
+    GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY: z.string().min(1).optional(),
   },
   clientPrefix: "NEVER_USED",
   client: {},
@@ -67,11 +72,11 @@ export const env = createEnv({
     WALLET_KEYS: {
       // The sdk expects a primitive type but we can overload it here to be an object
       WALLET_PRIVATE_KEY: process.env.WALLET_PRIVATE_KEY,
-      AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
-      AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
       AWS_KMS_KEY_ID: process.env.AWS_KMS_KEY_ID,
-      AWS_REGION: process.env.AWS_REGION,
     } as any,
+    AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
+    AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
+    AWS_REGION: process.env.AWS_REGION,
     THIRDWEB_SDK_SECRET_KEY: process.env.THIRDWEB_SDK_SECRET_KEY,
     THIRDWEB_API_ORIGIN: process.env.THIRDWEB_API_ORIGIN,
     POSTGRES_HOST: process.env.POSTGRES_HOST,
@@ -94,6 +99,13 @@ export const env = createEnv({
     MINED_TX_CRON_SCHEDULE: process.env.MINED_TX_CRON_SCHEDULE,
     MIN_TX_TO_CHECK_FOR_MINED_STATUS:
       process.env.MIN_TX_TO_CHECK_FOR_MINED_STATUS,
+    GCP_PROJECT_ID: process.env.GCP_PROJECT_ID,
+    GCP_KEY_RING_ID: process.env.GCP_KEY_RING_ID,
+    GCP_LOCATION_ID: process.env.GCP_LOCATION_ID,
+    GOOGLE_APPLICATION_CREDENTIAL_EMAIL:
+      process.env.GOOGLE_APPLICATION_CREDENTIAL_EMAIL,
+    GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY:
+      process.env.GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY,
   },
   onValidationError: (error: ZodError) => {
     if ("WALLET_KEYS" in error.format()) {

core/helpers/index.ts

@@ -1,3 +1,14 @@
+import { env } from "../env";
+
+const AWS_ACCESS_KEY_ID = env.AWS_ACCESS_KEY_ID;
+const AWS_SECRET_ACCESS_KEY = env.AWS_SECRET_ACCESS_KEY;
+const AWS_REGION = env.AWS_REGION;
+
+const WALLET_PRIVATE_KEY =
+  "WALLET_PRIVATE_KEY" in env.WALLET_KEYS
+    ? env.WALLET_KEYS.WALLET_PRIVATE_KEY
+    : undefined;
+
 export const isValidHttpUrl = (urlString: string): boolean => {
   let url;
 
@@ -9,3 +20,25 @@ export const isValidHttpUrl = (urlString: string): boolean => {
 
   return url.protocol === "http:" || url.protocol === "https:";
 };
+
+export const getInstanceAdminWalletType = (): string => {
+  if (WALLET_PRIVATE_KEY) {
+    return "ppk";
+  }
+
+  if (AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_REGION) {
+    return "aws_kms";
+  }
+
+  // ToDo GCP KMS
+  return "";
+};
+
+export const getWalletBackUpType = (): string => {
+  if (AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_REGION) {
+    return "aws_kms";
+  }
+
+  // ToDo GCP KMS
+  return "ppk";
+};

core/index.ts

@@ -1,4 +1,5 @@
 export * from "./database/dbConnect";
+export * from "./database/dbOperation";
 export * from "./database/dbPrereqs";
 export * from "./env";
 export * from "./error/customError";

core/interfaces/index.ts

@@ -4,4 +4,10 @@ export interface WalletData {
   lastUsedNonce: number;
   blockchainNonce: number;
   lastSyncedTimestamp: Date;
+  walletType: string;
+  awsKmsKeyId?: string;
+  gcpKmsKeyId?: string;
+  awsKmsKeyArn?: string;
+  gcpKmsKeyRingId?: string;
+  gcpKmsKeyVersion?: string;
 }