GCP KMS + other updates (#105)

* added web3api_overrides example

* GCP KMS implemented

* updated getAll

* updated env variable naming

* added wallet add end-point

* updated GCP env vars needed

* updated readme & added aws,google kms doc

* updated .env.example & update google kms doc

Author: farhanW3

.env.example

@@ -3,11 +3,22 @@
 # Either a private key
 WALLET_PRIVATE_KEY=<your_admin_wallet_private_key>
 # Or from AWS KMS
+AWS_KMS_KEY_ID=<your_aws_kms_key_id>
 AWS_ACCESS_KEY_ID=<your_aws_access_key_id>
 AWS_SECRET_ACCESS_KEY=<your_aws_secret_access_key>
-AWS_KMS_KEY_ID=<your_aws_kms_key_id>
 AWS_REGION=<your_aws_region>
 
+# Or from GOOGLE KMS
+# Required for Google Auth
+GOOGLE_APPLICATION_CREDENTIAL_EMAIL=<client_email_from_download_service_account_json>
+GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY=<private_key_from_download_service_account_json>
+# Required for Google KMS
+GOOGLE_APPLICATION_PROJECT_ID=<google_project_id>
+GOOGLE_KMS_KEY_RING_ID=<key_ring_id>
+GOOGLE_KMS_LOCATION_ID=<location_of_key_ring>
+GOOGLE_KMS_CRYPTO_KEY_ID=<kms_key_id> # If created on Google Console
+
+
 # THIRDWEB SDK SECRET KEY [Required]
 # -----------------------
 # Obtain an Secret Key from thirdweb.com/dashboard
@@ -62,5 +73,4 @@ BENCHMARK_POST_BODY='{
     "args": ["0x1946267d81Fb8aDeeEa28e6B98bcD446c8248473", 100000]
 }'
 BENCHMARK_CONCURRENCY=10
-BENCHMARK_REQUESTS=10
-
+BENCHMARK_REQUESTS=10

.github/aws_kms_how_to.md

@@ -0,0 +1,33 @@
+Web3-API supports AWS KMS for signing & sending transactions over any EVM chain. This is a guide on how to set up AWS KMS for Web3-API.
+
+### Steps to set up AWS KMS
+
+1. Create IAM user with programmatic access, see [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html#id_users_create_console) for more details.
+2. Add create, get, read permission to KMS, see [here](https://docs.aws.amazon.com/kms/latest/developerguide/control-access.html) for more details.
+3. Create a AWS KMS key, see [here](https://docs.aws.amazon.com/kms/latest/developerguide/create-keys.html) for more details. or, you can use the `/wallet/create` to create a key.
+
+NOTE:
+
+If you are creating the key yourself on AWS KMS Console, then please select the below config to create a key with sign permission.
+
+```
+Key Type: Asymmetric
+Key Spec: ECC_SECG_P256K1
+Key Usage: Sign and verify
+```
+
+Once you create the key above, you can use `/wallet/add` and send details on the end-point to create the wallet
+
+### Set up Web3-API with AWS KMS
+
+Create a `.env` file in the root directory of the project and add the below details.
+
+```
+# Required for AWS Auth
+AWS_ACCESS_KEY_ID=<aws_access_key_id>
+AWS_SECRET_ACCESS_KEY=<aws_secret_access_key>
+AWS_REGION=<aws_region>
+
+# Required for AWS KMS Admin Wallet
+AWS_KMS_KEY_ID=<kms_key_id>
+```

.github/google_kms_how_to.md

@@ -0,0 +1,32 @@
+Web3-API supports Google KMS for signing & sending transactions over any EVM chain. This is a guide on how to set up Google KMS for Web3-API.
+
+### Steps to set up Google KMS
+
+1. Enable Google KMS API for your Google project, see [here](https://cloud.google.com/kms/docs/create-encryption-keys#before-you-begin) for more details.
+2. Create a Service Account (here)[https://cloud.google.com/iam/docs/service-accounts-create] and create a key under this service account and download the JSON file. This JSON file details will be used to authenticate with Google KMS.
+3. Add the below permissions to the service account created in step 2.
+
+```
+Cloud KMS Admin
+Cloud KMS CryptoKey Signer/Verifier
+```
+
+4. Create a keyring in Google KMS, see [here](https://cloud.google.com/kms/docs/create-key-ring) for more details.
+
+Optional: Create a key in the keyring, see [here](https://cloud.google.com/kms/docs/create-key) for more details. or, you can use the `/wallet/create` to create a key in the keyring.
+
+### Set up Web3-API with Google KMS
+
+Create a `.env` file in the root directory of the project and add the below details.
+
+```
+# Required for Google Auth
+GOOGLE_APPLICATION_CREDENTIAL_EMAIL=<client_email_from_download_service_account_json>
+GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY=<private_key_from_download_service_account_json>
+
+# Required for Google KMS
+GOOGLE_APPLICATION_PROJECT_ID=<google_project_id>
+GOOGLE_KMS_KEY_RING_ID=<key_ring_id>
+GOOGLE_KMS_LOCATION_ID=<location_of_key_ring>
+GOOGLE_KMS_CRYPTO_KEY_ID=<kms_key_id> # If created on Google Console
+```

README.md

@@ -82,13 +82,11 @@ There are multiple ways to setup a wallet for Web3-API using the below methods:
 
 #### AWS KMS Wallet
 
-1. Get the AWS KMS Support Variables which can be found in `.env.example` file
-2. Update the AWS KMS ENV Variables with the correct values on `.env` file
+Read More on [AWS KMS How To](./.github/aws_kms_how_to.md)
 
-- `AWS_ACCESS_KEY_ID` : AWS Access Key
-- `AWS_SECRET_ACCESS_KEY` : AWS Secret Access Key
-- `AWS_REGION` : AWS KMS Key Region
-- `AWS_KMS_KEY_ID` : Needs to have the full ARN
+#### Google KMS Wallet
+
+Read More on [Google KMS How To](./.github/google_kms_how_to.md)
 
 ### Advance Setup : PostgreSQL DB
 

core/database/dbOperation.ts

@@ -15,6 +15,10 @@ interface WalletExtraData {
   walletType?: string;
   gcpKmsKeyId?: string;
   gcpKmsKeyRingId?: string;
+  gcpKmsLocationId?: string;
+  gcpKmsKeyVersionId?: string;
+  gcpKmsProjectId?: string;
+  gcpKmsResourcePath?: string;
 }
 
 export const insertIntoWallets = async (
@@ -89,19 +93,28 @@ export const addWalletDataWithSupportChainsNonceToDB = async (
   extraTableData?: WalletExtraData,
 ): Promise<void> => {
   try {
-    server.log.info("Setting up wallet Table");
+    server.log.info(
+      `Setting up wallet Table for walletType ${extraTableData?.walletType}, walletAddress ${walletAddress}`,
+    );
     const supportedChains = await getSupportedChains();
     const promises = supportedChains.map(async (chain) => {
       try {
         const { slug } = chain;
         let lastUsedNonce = -1;
-        let walletType = isWeb3APIInitWallet
+        let walletType = extraTableData?.walletType
+          ? extraTableData?.walletType
+          : isWeb3APIInitWallet
           ? getInstanceAdminWalletType()
           : getWalletBackUpType();
         const sdk = await getSDK(slug, {
           walletAddress,
           walletType,
           awsKmsKeyId: extraTableData?.awsKmsKeyId,
+          gcpKmsKeyId: extraTableData?.gcpKmsKeyId,
+          gcpKmsKeyRingId: extraTableData?.gcpKmsKeyRingId,
+          gcpKmsLocationId: extraTableData?.gcpKmsLocationId,
+          gcpKmsKeyVersionId: extraTableData?.gcpKmsKeyVersionId,
+          gcpKmsResourcePath: extraTableData?.gcpKmsResourcePath,
         });
         walletAddress =
           (await sdk.getSigner()?.getAddress())?.toLowerCase() ?? "";

core/database/sql-schemas/wallets.sql

@@ -8,9 +8,11 @@ CREATE TABLE IF NOT EXISTS wallets (
     "lastUsedNonce" BIGINT NOT NULL,
     "awsKmsKeyId" VARCHAR(255),
     "awsKmsArn" VARCHAR(255),
-    -- "gcpKmsKeyRingId" VARCHAR(50),
-    -- "gcpKmsKeyId" VARCHAR(50),
-    -- "gcpKmsKeyVersion" VARCHAR(20),
+    "gcpKmsKeyRingId" VARCHAR(50),
+    "gcpKmsKeyId" VARCHAR(50),
+    "gcpKmsKeyVersionId" VARCHAR(20),
+    "gcpKmsLocationId" VARCHAR(20),
+    "gcpKmsResourcePath" TEXT,
     PRIMARY KEY ("walletAddress", "chainId")
 );
 
@@ -22,7 +24,11 @@ ALTER COLUMN "lastUsedNonce" TYPE BIGINT,
 ADD COLUMN IF NOT EXISTS "awsKmsKeyId" VARCHAR(255),
 ADD COLUMN IF NOT EXISTS "awsKmsArn" VARCHAR(255),
 ADD COLUMN IF NOT EXISTS "slug" VARCHAR(255),
-DROP COLUMN IF EXISTS "chainName";
--- ADD COLUMN IF NOT EXISTS "gcpKmsKeyRingId" VARCHAR(50),
--- ADD COLUMN IF NOT EXISTS "gcpKmsKeyId" VARCHAR(50),
--- ADD COLUMN IF NOT EXISTS "gcpKmsKeyVersion" VARCHAR(20);
+DROP COLUMN IF EXISTS "chainName",
+ADD COLUMN IF NOT EXISTS "gcpKmsKeyRingId" VARCHAR(50),
+ADD COLUMN IF NOT EXISTS "gcpKmsKeyId" VARCHAR(50),
+ADD COLUMN IF NOT EXISTS "gcpKmsKeyVersionId" VARCHAR(20),
+ADD COLUMN IF NOT EXISTS "gcpKmsLocationId" VARCHAR(20),
+DROP COLUMN IF EXISTS "gcpKmsKeyVersion",
+DROP COLUMN IF EXISTS "gcpKmsKeyPath",
+ADD COLUMN IF NOT EXISTS "gcpKmsResourcePath" TEXT;

core/env.ts

@@ -31,6 +31,9 @@ export const env = createEnv({
       z.object({
         AWS_KMS_KEY_ID: z.string().min(1),
       }),
+      z.object({
+        GOOGLE_KMS_KEY_ID: z.string().min(1).optional(),
+      }),
     ]),
     AWS_ACCESS_KEY_ID: z.string().min(1).optional(),
     AWS_SECRET_ACCESS_KEY: z.string().min(1).optional(),
@@ -58,9 +61,10 @@ export const env = createEnv({
     MINED_TX_CRON_ENABLED: boolSchema("true"),
     MINED_TX_CRON_SCHEDULE: z.string().default("*/5 * * * * *"),
     MIN_TX_TO_CHECK_FOR_MINED_STATUS: z.coerce.number().default(50),
-    GCP_PROJECT_ID: z.string().min(1).optional(),
-    GCP_KEY_RING_ID: z.string().min(1).optional(),
-    GCP_LOCATION_ID: z.string().min(1).optional(),
+    GOOGLE_APPLICATION_PROJECT_ID: z.string().min(1).optional(),
+    GOOGLE_KMS_KEY_RING_ID: z.string().min(1).optional(),
+    GOOGLE_KMS_LOCATION_ID: z.string().min(1).optional(),
+    GOOGLE_KMS_KEY_VERSION_ID: z.string().min(1).optional(),
     GOOGLE_APPLICATION_CREDENTIAL_EMAIL: z.string().min(1).optional(),
     GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY: z.string().min(1).optional(),
   },
@@ -73,6 +77,7 @@ export const env = createEnv({
       // The sdk expects a primitive type but we can overload it here to be an object
       WALLET_PRIVATE_KEY: process.env.WALLET_PRIVATE_KEY,
       AWS_KMS_KEY_ID: process.env.AWS_KMS_KEY_ID,
+      GOOGLE_KMS_KEY_ID: process.env.GOOGLE_KMS_KEY_ID,
     } as any,
     AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
     AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
@@ -99,9 +104,10 @@ export const env = createEnv({
     MINED_TX_CRON_SCHEDULE: process.env.MINED_TX_CRON_SCHEDULE,
     MIN_TX_TO_CHECK_FOR_MINED_STATUS:
       process.env.MIN_TX_TO_CHECK_FOR_MINED_STATUS,
-    GCP_PROJECT_ID: process.env.GCP_PROJECT_ID,
-    GCP_KEY_RING_ID: process.env.GCP_KEY_RING_ID,
-    GCP_LOCATION_ID: process.env.GCP_LOCATION_ID,
+    GOOGLE_APPLICATION_PROJECT_ID: process.env.GOOGLE_APPLICATION_PROJECT_ID,
+    GOOGLE_KMS_KEY_RING_ID: process.env.GOOGLE_KMS_KEY_RING_ID,
+    GOOGLE_KMS_LOCATION_ID: process.env.GOOGLE_KMS_LOCATION_ID,
+    GOOGLE_KMS_KEY_VERSION_ID: process.env.GOOGLE_KMS_KEY_VERSION_ID,
     GOOGLE_APPLICATION_CREDENTIAL_EMAIL:
       process.env.GOOGLE_APPLICATION_CREDENTIAL_EMAIL,
     GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY:

core/helpers/index.ts

@@ -31,14 +31,25 @@ export const getInstanceAdminWalletType = (): string => {
   }
 
   // ToDo GCP KMS
-  return "";
+  return "gcp_kms";
 };
 
 export const getWalletBackUpType = (): string => {
   if (AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_REGION) {
     return "aws_kms";
   }
 
+  if (
+    env.GOOGLE_APPLICATION_CREDENTIAL_EMAIL &&
+    env.GOOGLE_APPLICATION_CREDENTIAL_PRIVATE_KEY &&
+    env.GOOGLE_APPLICATION_PROJECT_ID &&
+    env.GOOGLE_KMS_KEY_RING_ID &&
+    env.GOOGLE_KMS_KEY_VERSION_ID &&
+    env.GOOGLE_KMS_LOCATION_ID
+  ) {
+    return "gcp_kms";
+  }
+
   // ToDo GCP KMS
   return "ppk";
 };

core/interfaces/index.ts

@@ -9,5 +9,7 @@ export interface WalletData {
   gcpKmsKeyId?: string;
   awsKmsKeyArn?: string;
   gcpKmsKeyRingId?: string;
-  gcpKmsKeyVersion?: string;
+  gcpKmsKeyVersionId?: string;
+  gcpKmsLocationId?: string;
+  gcpKmsProjectId?: string;
 }