feat: Redis Workers POC (#467)

* Added beta packages & db

* updated ingest layer to write to bullMQ (ioredis)

* removed extra logs

* committed graceful shutdown. Need to revist

* updated webhooks to work based on worker. updated req ingestion

* updates

* updated to use queueTx() and allow all write end-points to work

* updated webhook flow

* Added beta GH workflow + minor updates

* updates for idempotencyKey

* updates for txenum

* wip

* simpify workers

* reuse thirdweb client

* brought back webhook pg listener. updated setex

* updates to get things stable

* wip

* wip

* Packages updates (#473)

* Introduce strict Transaction types

* clean up webhook code

* "fix" signature before encoding relayer execute call (#474)

* fix: add errorMessage to query to get queued txs (#475)

* fix: add errorMessage to query to get queued txs

* remove unused files

* update thirdweb package

* getWalletBalance fix

* chore(deps): bump express from 4.18.2 to 4.19.2 (#470)

Bumps [express](https://github.com/expressjs/express) from 4.18.2 to 4.19.2.
- [Release notes](https://github.com/expressjs/express/releases)
- [Changelog](https://github.com/expressjs/express/blob/master/History.md)
- [Commits](expressjs/express@4.18.2...4.19.2)

---
updated-dependencies:
- dependency-name: express
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Updated Mined Tx flow (#476)

* Bump follow-redirects from 1.15.2 to 1.15.6 (#448)

Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.2 to 1.15.6.
- [Release notes](https://github.com/follow-redirects/follow-redirects/releases)
- [Commits](follow-redirects/follow-redirects@v1.15.2...v1.15.6)

---
updated-dependencies:
- dependency-name: follow-redirects
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Cache webhooks

* fix webhooks

* remove debug lines

* serialize worker data

* naming conflict

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: farhanW3 <farhan@thirdweb.com>
Co-authored-by: farhanW3 <132962163+farhanW3@users.noreply.github.com>
Co-authored-by: Joaquim Verges <joaquim.verges@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: 4 people

.github/workflows/beta.yml

@@ -0,0 +1,43 @@
+name: Build on beta branch
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: beta # checkout the main branch to build nightly
+
+      - name: Check Disk Space Before Build
+        run: df -h
+
+      - name: Docker Prune
+        run: docker system prune -af
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Push Docker Image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          target: prod
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: thirdweb/engine:beta-nightly
+          build-args: |
+            ENGINE_VERSION=beta-nightly
+
+      - name: Check Disk Space After Build
+        run: df -h

.vscode/settings.json

@@ -7,5 +7,6 @@
   "editor.formatOnSave": true,
   "eslint.rules.customizations": [{ "rule": "*", "severity": "warn" }],
   "typescript.tsdk": "node_modules/typescript/lib",
-  "typescript.enablePromptUseWorkspaceTsdk": true
+  "typescript.enablePromptUseWorkspaceTsdk": true,
+  "cSpell.words": ["idempotency"]
 }

docker-compose-infra.yml

@@ -20,5 +20,29 @@ services:
           cpus: "2"
           memory: 2G
 
+  redis:
+    container_name: redis
+    image: redis:latest
+    restart: always
+    ports:
+      - 6379:6379
+    volumes:
+      - redis_data:/data
+
+  bullboard:
+    container_name: bullboard
+    image: deadly0/bull-board:latest
+    restart: always
+    ports:
+      - 3000:3000
+    environment:
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      REDIS_USE_TLS: "false"
+      BULL_PREFIX: bull
+    depends_on:
+      - redis
+
 volumes:
   psql_db:
+  redis_data:

docker-compose.yml

@@ -15,6 +15,13 @@ services:
         reservations:
           cpus: "2"
           memory: 2G
+  redis:
+    image: redis:latest
+    container_name: redis_container
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
 
   engine:
     build:
@@ -38,3 +45,4 @@ services:
 
 volumes:
   db_data:
+  redis_data:

package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "docker": "docker compose --env-file ./.env up --remove-orphans",
     "docker:build": "docker compose build --no-cache",
-    "dev": "yarn dev:infra && yarn dev:db && yarn dev:run",
+    "dev": "yarn dev:infra && yarn dev:run",
     "dev:infra": "docker compose -f ./docker-compose-infra.yml up -d",
     "dev:db": "yarn prisma:setup:dev",
     "dev:run": "nodemon --watch 'src/**/*.ts' --exec 'npx tsx ./src/index.ts' --files src/index.ts",
@@ -47,24 +47,29 @@
     "@types/base-64": "^1.0.2",
     "base-64": "^1.0.0",
     "body-parser": "^1.20.2",
+    "bullmq": "^5.4.2",
     "cookie": "^0.5.0",
     "cookie-parser": "^1.4.6",
     "copyfiles": "^2.4.1",
     "cron-parser": "^4.9.0",
     "crypto-js": "^4.2.0",
     "dotenv": "^16.0.3",
     "ethers": "5",
+    "ethers-aws-kms-signer": "^1.3.2",
+    "ethers-gcp-kms-signer": "^1.1.6",
     "fastify": "^4.15.0",
     "fastify-plugin": "^4.5.0",
     "http-status-codes": "^2.2.0",
+    "ioredis": "^5.3.2",
     "knex": "^3.1.0",
     "mnemonist": "^0.39.8",
     "node-cron": "^3.0.2",
     "pg": "^8.11.3",
     "pino": "^8.15.1",
     "pino-pretty": "^10.0.0",
     "prisma": "^5.2.0",
-    "thirdweb": "^5.0.0-beta-ca68bc77e74f594360b4da1e9e77793b66cfb12a-20240323045412",
+    "superjson": "^2.2.1",
+    "thirdweb": "^5.1.0",
     "uuid": "^9.0.1",
     "viem": "^1.14.0",
     "zod": "^3.21.4"

src/db/transactions/cleanTxs.ts

@@ -1,13 +1,8 @@
 import { Transactions } from "@prisma/client";
-import { Static } from "@sinclair/typebox";
 import { ContractExtension } from "../../schema/extension";
 import { PrismaTransaction } from "../../schema/prisma";
-import {
-  TransactionStatus,
-  transactionResponseSchema,
-} from "../../server/schemas/transaction";
+import { TransactionStatus } from "../../server/schemas/transaction";
 import { getPrismaWithPostgresTx } from "../client";
-import { cleanTxs } from "./cleanTxs";
 
 interface GetAllTxsParams {
   pgtx?: PrismaTransaction;
@@ -18,7 +13,7 @@ interface GetAllTxsParams {
 }
 
 interface GetAllTxsResponse {
-  transactions: Static<typeof transactionResponseSchema>[];
+  transactions: Transactions[];
   totalCount: number;
 }
 
@@ -99,7 +94,7 @@ export const getAllTxs = async ({
   const [totalCount, txs] = await Promise.all([totalCountPromise, txsPromise]);
 
   return {
-    transactions: cleanTxs(txs),
+    transactions: txs,
     totalCount: totalCount,
   };
 };

src/db/transactions/getAllTxs.ts

@@ -1,6 +1,5 @@
 import { PrismaTransaction } from "../../schema/prisma";
 import { getPrismaWithPostgresTx } from "../client";
-import { cleanTxs } from "./cleanTxs";
 
 interface GetAllTxsByWalletParams {
   pgtx?: PrismaTransaction;
@@ -14,7 +13,7 @@ export const getAllTxsByWallet = async ({
 }: GetAllTxsByWalletParams) => {
   const prisma = getPrismaWithPostgresTx(pgtx);
 
-  const txs = await prisma.transactions.findMany({
+  return await prisma.transactions.findMany({
     where: {
       fromAddress: walletAddress.toLowerCase(),
     },
@@ -24,6 +23,4 @@ export const getAllTxsByWallet = async ({
       },
     ],
   });
-
-  return cleanTxs(txs);
 };

src/db/transactions/getAllTxsByWallet.ts

@@ -23,6 +23,7 @@ export const getQueuedTxs = async ({ pgtx }: GetQueuedTxsParams = {}): Promise<
     "sentAt" IS NULL
     AND "minedAt" IS NULL
     AND "cancelledAt" IS NULL
+    AND "errorMessage" IS NULL
   ORDER BY
     "queuedAt"
   ASC

src/db/transactions/getQueuedTxs.ts

@@ -1,8 +1,10 @@
 import { Static } from "@sinclair/typebox";
 import { PrismaTransaction } from "../../schema/prisma";
-import { transactionResponseSchema } from "../../server/schemas/transaction";
+import {
+  toTransactionResponse,
+  transactionResponseSchema,
+} from "../../server/schemas/transaction";
 import { getPrismaWithPostgresTx } from "../client";
-import { cleanTxs } from "./cleanTxs";
 
 interface GetTxByIdParams {
   queueId: string;
@@ -24,11 +26,5 @@ export const getTxById = async ({
       id: queueId,
     },
   });
-
-  if (!tx) {
-    return null;
-  }
-
-  const [cleanedTx] = cleanTxs([tx]);
-  return cleanedTx;
+  return tx ? toTransactionResponse(tx) : null;
 };

src/db/transactions/getTxById.ts

@@ -1,6 +1,6 @@
+import { Transactions } from "@prisma/client";
 import { PrismaTransaction } from "../../schema/prisma";
 import { getPrismaWithPostgresTx } from "../client";
-import { cleanTxs } from "./cleanTxs";
 
 interface GetTxsByGroupIdParams {
   pgtx?: PrismaTransaction;
@@ -10,17 +10,12 @@ interface GetTxsByGroupIdParams {
 export const getTxsByGroupId = async ({
   pgtx,
   groupId,
-}: GetTxsByGroupIdParams) => {
+}: GetTxsByGroupIdParams): Promise<Transactions[]> => {
   const prisma = getPrismaWithPostgresTx(pgtx);
   const txs = await prisma.transactions.findMany({
     where: {
       groupId,
     },
   });
-
-  if (!txs) {
-    return [];
-  }
-
-  return cleanTxs(txs);
+  return txs;
 };