Skip to content

Commit be977b7

Browse files
feat(benchmark): Add quantified benchmarking for nonce management (#88)
* feat(benchmark): move benchmarking to ts script for additional configurations * chore: remove unused env variable * chore: add .env.benchmark to gitignore chore(server): sort imports * feat(server): add txProcessedTimestamp to returned transaction object * feat(worker): override gas fee setting to allow for sending of transaction on polygon * feat(benchmark): Add better info for initial send reqeust feat(benchmark): Add support for checking transaction status times * docs(core): add description for benchmark file * chore(core): remove logs from getFeeData chore(worker): remove fee data info for now bc it's hacky and won't work with non eip1559 evm chains * chore(benchmark): Update timings to reflect + print status for easy viewing * chore: update @thirdweb-dev/sdk and /chains * chore(benchmark): add benchmarking env file support * feat(benchmark): update write vars * docs(benchmark): add docs to readme on how to run local benchmark * merged changes * Added blockNumber & txMinedTimestamp column to transactions table * updated status response example * updated Query & tx selection based on receipt * updated resposne for tx statuse end-point * updated benchmark with Procssing Time, Mined Time --------- Signed-off-by: farhanW3 <132962163+farhanW3@users.noreply.github.com> Co-authored-by: farhanW3 <farhan@thirdweb.com> Co-authored-by: farhanW3 <132962163+farhanW3@users.noreply.github.com>
1 parent 7c42ac6 commit be977b7

File tree

24 files changed

+843
-87
lines changed

24 files changed

+843
-87
lines changed

.env.benchmark.example

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
THIRDWEB_SDK_SECRET_KEY=
2+
3+
# benchmark vars
4+
NODE_ENV=benchmark
5+
BENCHMARK_HOST='http://127.0.0.1:3005'
6+
BENCHMARK_URL_PATH='/contract/polygon/0x01De66609582B874FA34ab288859ACC4592aec04/write'
7+
BENCHMARK_POST_BODY='{
8+
"function_name": "mintTo",
9+
"args": ["0xCF3D06a19263976A540CFf8e7Be7b026801C52A6", "0","", "1"]
10+
}'
11+
# Total request has to be more than the total concurrency
12+
BENCHMARK_CONCURRENCY=2
13+
BENCHMARK_REQUESTS=2
14+
SAMPLE_EDITION_CONTRACT_INFO='{
15+
"contractMetadata": {
16+
"name": "TW WEB3API",
17+
"description": "thirdweb web3API sample collection for testing",
18+
"image": "ipfs://QmYxT4LnK8sqLupjbS6eRvu1si7Ly2wFQAqFebxhWntcf6",
19+
"external_link": "",
20+
"app_uri": "",
21+
"seller_fee_basis_points": 0,
22+
"fee_recipient": "0xCF3D06a19263976A540CFf8e7Be7b026801C52A6",
23+
"symbol": "TWT",
24+
"platform_fee_basis_points": 0,
25+
"platform_fee_recipient": "0xCF3D06a19263976A540CFf8e7Be7b026801C52A6",
26+
"primary_sale_recipient": "0xCF3D06a19263976A540CFf8e7Be7b026801C52A6",
27+
"trusted_forwarders": []
28+
}
29+
}'

.env.example

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ CHAIN_OVERRIDES=<path_or_url_to_file>
5454
# from "http://example1.com" or from a subdomain of "example2.com".
5555
ACCESS_CONTROL_ALLOW_ORIGIN=*
5656

57-
5857
# benchmark vars [Optional]
5958
BENCHMARK_HOST='http://localhost:3005'
6059
BENCHMARK_URL_PATH='/contract/mumbai/0xc8be6265C06aC376876b4F62670adB3c4d72EABA/write'
@@ -64,3 +63,4 @@ BENCHMARK_POST_BODY='{
6463
}'
6564
BENCHMARK_CONCURRENCY=10
6665
BENCHMARK_REQUESTS=10
66+

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ web_modules/
7474

7575
# dotenv environment variable files
7676
.env
77+
.env.benchmark
7778
.env.development.local
7879
.env.test.local
7980
.env.production.local

Makefile

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
SHELL := /bin/bash
44

55
export NODE_ENV=testing
6-
export THIRDWEB_API_KEY=TEST
76
export POSTGRES_HOST=localhost
87
export POSTGRES_DATABASE_NAME=postgres
98
export POSTGRES_USER=postgres

README.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,16 @@ The API defaults to `http://localhost:3005`
205205

206206
</details>
207207

208+
## Local Benchmarking
209+
210+
As a way to support quantifying the robustness of our system, we have added benchmarking. Benchmark results may vary based on the machine that is being used.
211+
212+
To run the benchmark:
213+
214+
1. Run local server with `yarn dev`
215+
1. Set-up `.env.benchmark` (For sensible defaults: `cp .env.benchmark.example .env.benchmark`)
216+
1. Run benchmark in a separate terminal with `yarn benchmark`
217+
208218
## Contributing
209219

210220
We welcome contributions from all developers, regardless of experience level. If you are interested in contributing, please read our [Contributing Guide](./.github/contributing.md) where you'll learn how the repo works, how to test your changes, and how to submit a pull request.

core/database/dbConnect.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ export const connectWithDatabase = async (): Promise<Knex> => {
8989
};
9090

9191
// Set the appropriate databse client package
92-
let dbClientPackage: any;
92+
let dbClientPackage: typeof pg;
9393
switch (dbClient) {
9494
case "pg":
9595
dbClientPackage = pg;

core/database/sql-schemas/transactions.sql

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,6 @@ ALTER COLUMN "maxFeePerGas" TYPE VARCHAR(255),
4040
ALTER COLUMN "txType" TYPE VARCHAR(255),
4141
ADD COLUMN IF NOT EXISTS "deployedContractAddress" VARCHAR(255),
4242
ADD COLUMN IF NOT EXISTS "contractType" VARCHAR(255),
43-
ADD COLUMN IF NOT EXISTS "errorMessage" TEXT DEFAULT NULL;
43+
ADD COLUMN IF NOT EXISTS "errorMessage" TEXT DEFAULT NULL,
44+
ADD COLUMN IF NOT EXISTS "txMinedTimestamp" TIMESTAMP,
45+
ADD COLUMN IF NOT EXISTS "blockNumber" BIGINT;

core/services/benchmark.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// ! Winston Notes: This file is currently not in used.
2+
// Keeping it around for when we might want to track specific function timings.
3+
// Note that you cannot track functions in worker and output timings in server.
4+
// Tracking and outputting timings must be done in the same docker image.
5+
// Might release this as my own npm package and bring it in as a dependency in the future
6+
import { randomUUID } from "crypto";
7+
import { PerformanceObserver, performance } from "perf_hooks";
8+
9+
export const watchPerformance = (
10+
onNewEntry: (entry: PerformanceEntry) => void,
11+
) => {
12+
const performanceObserver = new PerformanceObserver((items) => {
13+
items.getEntries().forEach((entry) => {
14+
onNewEntry(entry);
15+
});
16+
});
17+
performanceObserver.observe({ entryTypes: ["measure"], buffered: true });
18+
};
19+
20+
export const timeFunction = <
21+
F extends (...args: any) => any,
22+
Args = F extends (args: infer A) => any ? A : never,
23+
>(
24+
fn: F,
25+
resultTag: (args: Awaited<ReturnType<F>>) => string,
26+
) => {
27+
const result = async (args: Args): Promise<Awaited<ReturnType<F>>> => {
28+
const marker = randomUUID();
29+
const markerStart = `${marker}-start`;
30+
const markerEnd = `${marker}-end`;
31+
32+
performance.mark(markerStart);
33+
const fnResult = await fn(args);
34+
performance.mark(markerEnd);
35+
36+
const measureName = resultTag(fnResult);
37+
performance.measure(measureName, markerStart, markerEnd);
38+
return fnResult;
39+
};
40+
return result;
41+
};

core/services/blockchain.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,13 @@ export const getWalletNonce = async (
1414
throw error;
1515
}
1616
};
17+
18+
export const getFeeData = async (provider: providers.Provider) => {
19+
try {
20+
const feeData = await provider.getFeeData();
21+
22+
return feeData;
23+
} catch (error) {
24+
throw error;
25+
}
26+
};

package.json

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"lint": "eslint 'server/**/*.ts'",
2424
"lint:fix": "eslint --fix 'server/**/*.ts'",
2525
"test": "make test-evm",
26-
"test:all": "mocha --exit"
26+
"test:all": "mocha --exit",
27+
"benchmark": "ts-node ./scripts/benchmark/index.ts"
2728
},
2829
"dependencies": {
2930
"@fastify/cookie": "^8.3.0",
@@ -59,6 +60,7 @@
5960
"devDependencies": {
6061
"@nomiclabs/hardhat-ethers": "^2.2.3",
6162
"@swc/core": "^1.3.41",
63+
"@types/autocannon": "^7.9.1",
6264
"@types/chai": "^4.3.5",
6365
"@types/cookie": "^0.5.1",
6466
"@types/express": "^4.17.17",
@@ -71,6 +73,7 @@
7173
"@types/ws": "^8.5.5",
7274
"@typescript-eslint/eslint-plugin": "^5.55.0",
7375
"@typescript-eslint/parser": "^5.55.0",
76+
"autocannon": "^7.12.0",
7477
"chai": "^4.3.7",
7578
"eslint": "^8.36.0",
7679
"eslint-config-prettier": "^8.7.0",

0 commit comments

Comments
 (0)