Skip to content

Commit 6e554a8

Browse files
iuwqyiriuwqyir
authored
Merge pull request #240 from thirdweb-dev/07-21-update_clickhouse_schemas_to_latest_state
update clickhouse schemas to latest state
2 parents 513dd51 + f4d13a5 commit 6e554a8

File tree

4 files changed

+240
-4
lines changed

4 files changed

+240
-4
lines changed

‎internal/tools/clickhouse_create_logs_table.sql

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,18 @@ CREATE TABLE IF NOT EXISTS logs (
2222
INDEX idx_topic1 topic_1 TYPE bloom_filter GRANULARITY 1,
2323
INDEX idx_topic2 topic_2 TYPE bloom_filter GRANULARITY 1,
2424
INDEX idx_topic3 topic_3 TYPE bloom_filter GRANULARITY 1,
25+
PROJECTION logs_chainid_topic0_address
26+
(
27+
SELECT *
28+
ORDER BY
29+
chain_id,
30+
topic_0,
31+
address,
32+
block_number,
33+
transaction_index,
34+
log_index
35+
)
2536
) ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
2637
ORDER BY (chain_id, block_number, transaction_hash, log_index)
27-
PARTITION BY chain_id;
38+
PARTITION BY chain_id
39+
SETTINGS deduplicate_merge_projection_mode = 'drop', lightweight_mutation_projection_mode = 'rebuild';

‎internal/tools/clickhouse_create_token_balances_mv.sql

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,20 @@ CREATE TABLE IF NOT EXISTS token_balances
55
`owner` FixedString(42),
66
`address` FixedString(42),
77
`token_id` UInt256,
8-
`balance` Int256
8+
`balance` Int256,
9+
PROJECTION address_projection
10+
(
11+
SELECT *
12+
ORDER BY
13+
token_type,
14+
chain_id,
15+
address,
16+
token_id
17+
)
918
)
1019
ENGINE = SummingMergeTree
11-
ORDER BY (token_type, chain_id, owner, address, token_id);
20+
ORDER BY (token_type, chain_id, owner, address, token_id)
21+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild';
1222

1323
CREATE MATERIALIZED VIEW IF NOT EXISTS single_token_transfers_mv TO token_balances AS
1424
SELECT chain_id, owner, address, token_type, token_id, sum(amount) as balance

‎internal/tools/clickhouse_create_token_transfers_mv.sql

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
CREATE TABLE IF NOT EXISTS token_transfers
2+
(
3+
`token_type` LowCardinality(String),
4+
`chain_id` UInt256,
5+
`token_address` FixedString(42),
6+
`from_address` FixedString(42),
7+
`to_address` FixedString(42),
8+
`block_number` UInt256,
9+
`block_timestamp` DateTime CODEC(Delta(4), ZSTD(1)),
10+
`transaction_hash` FixedString(66),
11+
`token_id` UInt256,
12+
`amount` UInt256,
13+
`log_index` UInt64,
14+
`sign` Int8 DEFAULT 1,
15+
`insert_timestamp` DateTime DEFAULT now(),
16+
PROJECTION from_address_projection
17+
(
18+
SELECT *
19+
ORDER BY
20+
chain_id,
21+
token_type,
22+
from_address,
23+
block_number,
24+
log_index
25+
),
26+
PROJECTION to_address_projection
27+
(
28+
SELECT *
29+
ORDER BY
30+
chain_id,
31+
token_type,
32+
to_address,
33+
block_number,
34+
log_index
35+
),
36+
PROJECTION transaction_hash_projection
37+
(
38+
SELECT *
39+
ORDER BY
40+
chain_id,
41+
token_type,
42+
transaction_hash,
43+
block_number,
44+
log_index
45+
)
46+
)
47+
ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
48+
PARTITION BY chain_id
49+
ORDER BY (chain_id, token_type, token_address, block_number, log_index)
50+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild';
51+
52+
CREATE MATERIALIZED VIEW IF NOT EXISTS logs_to_token_transfers TO token_transfers
53+
(
54+
`chain_id` UInt256,
55+
`token_address` FixedString(42),
56+
`from_address` String,
57+
`to_address` String,
58+
`token_type` String,
59+
`block_number` UInt256,
60+
`block_timestamp` DateTime,
61+
`transaction_hash` FixedString(66),
62+
`log_index` UInt64,
63+
`sign` Int8,
64+
`insert_timestamp` DateTime,
65+
`token_id` UInt256,
66+
`amount` UInt256
67+
)
68+
AS WITH
69+
transfer_logs AS
70+
(
71+
SELECT
72+
chain_id,
73+
address AS token_address,
74+
topic_0,
75+
topic_1,
76+
topic_2,
77+
topic_3,
78+
(topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef') AND (topic_3 = '') AS is_erc20,
79+
(topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef') AND (topic_3 != '') AS is_erc721,
80+
topic_0 IN ('0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62', '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb') AS is_erc1155,
81+
multiIf(is_erc20, 'erc20', is_erc721, 'erc721', 'erc1155') AS token_type,
82+
if(is_erc1155, concat('0x', substring(topic_2, 27, 40)), concat('0x', substring(topic_1, 27, 40))) AS from_address,
83+
if(is_erc1155, concat('0x', substring(topic_3, 27, 40)), concat('0x', substring(topic_2, 27, 40))) AS to_address,
84+
data,
85+
block_number,
86+
block_timestamp,
87+
transaction_hash,
88+
log_index,
89+
sign,
90+
insert_timestamp
91+
FROM default.logs
92+
WHERE topic_0 IN ('0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62', '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb')
93+
),
94+
batch_transfer_metadata AS
95+
(
96+
SELECT
97+
*,
98+
3 + (2 * 64) AS ids_length_idx,
99+
ids_length_idx + 64 AS ids_values_idx,
100+
reinterpretAsUInt64(reverse(unhex(substring(data, ids_length_idx, 64)))) AS ids_length,
101+
(ids_length_idx + 64) + (ids_length * 64) AS amounts_length_idx,
102+
reinterpretAsUInt64(reverse(unhex(substring(data, amounts_length_idx, 64)))) AS amounts_length,
103+
amounts_length_idx + 64 AS amounts_values_idx
104+
FROM transfer_logs
105+
WHERE (topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb') AND (length(topic_1) = 66) AND (length(topic_2) = 66) AND (length(topic_3) = 66) AND (length(data) != (258 + ((ids_length + amounts_length) * 64))) AND (ids_length = amounts_length)
106+
),
107+
batch_transfer_logs AS
108+
(
109+
SELECT
110+
*,
111+
arrayMap(x -> substring(data, ids_values_idx + ((x - 1) * 64), 64), range(1, toInt32(ids_length) + 1)) AS ids_hex,
112+
arrayMap(x -> substring(data, amounts_values_idx + ((x - 1) * 64), 64), range(1, toInt32(amounts_length) + 1)) AS amounts_hex
113+
FROM batch_transfer_metadata
114+
)
115+
SELECT
116+
chain_id,
117+
token_address,
118+
from_address,
119+
to_address,
120+
token_type,
121+
block_number,
122+
block_timestamp,
123+
transaction_hash,
124+
log_index,
125+
sign,
126+
insert_timestamp,
127+
multiIf(is_erc1155, reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))), is_erc721, reinterpretAsUInt256(reverse(unhex(substring(topic_3, 3, 64)))), toUInt256(0)) AS token_id,
128+
multiIf(is_erc20 AND (length(data) = 66), reinterpretAsUInt256(reverse(unhex(substring(data, 3)))), is_erc721, toUInt256(1), is_erc1155, if(length(data) = 130, reinterpretAsUInt256(reverse(unhex(substring(data, 67, 64)))), toUInt256(1)), toUInt256(0)) AS amount
129+
FROM transfer_logs
130+
WHERE topic_0 IN ('0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62')
131+
UNION ALL
132+
WITH
133+
transfer_logs AS
134+
(
135+
SELECT
136+
chain_id,
137+
address AS token_address,
138+
topic_0,
139+
topic_1,
140+
topic_2,
141+
topic_3,
142+
(topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef') AND (topic_3 = '') AS is_erc20,
143+
(topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef') AND (topic_3 != '') AS is_erc721,
144+
topic_0 IN ('0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62', '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb') AS is_erc1155,
145+
multiIf(is_erc20, 'erc20', is_erc721, 'erc721', 'erc1155') AS token_type,
146+
if(is_erc1155, concat('0x', substring(topic_2, 27, 40)), concat('0x', substring(topic_1, 27, 40))) AS from_address,
147+
if(is_erc1155, concat('0x', substring(topic_3, 27, 40)), concat('0x', substring(topic_2, 27, 40))) AS to_address,
148+
data,
149+
block_number,
150+
block_timestamp,
151+
transaction_hash,
152+
log_index,
153+
sign,
154+
insert_timestamp
155+
FROM default.logs
156+
WHERE topic_0 IN ('0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62', '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb')
157+
),
158+
batch_transfer_metadata AS
159+
(
160+
SELECT
161+
*,
162+
3 + (2 * 64) AS ids_length_idx,
163+
ids_length_idx + 64 AS ids_values_idx,
164+
reinterpretAsUInt64(reverse(unhex(substring(data, ids_length_idx, 64)))) AS ids_length,
165+
(ids_length_idx + 64) + (ids_length * 64) AS amounts_length_idx,
166+
reinterpretAsUInt64(reverse(unhex(substring(data, amounts_length_idx, 64)))) AS amounts_length,
167+
amounts_length_idx + 64 AS amounts_values_idx
168+
FROM transfer_logs
169+
WHERE (topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb') AND (length(topic_1) = 66) AND (length(topic_2) = 66) AND (length(topic_3) = 66) AND (length(data) != (258 + ((ids_length + amounts_length) * 64))) AND (ids_length = amounts_length)
170+
),
171+
batch_transfer_logs AS
172+
(
173+
SELECT
174+
*,
175+
arrayMap(x -> substring(data, ids_values_idx + ((x - 1) * 64), 64), range(1, toInt32(ids_length) + 1)) AS ids_hex,
176+
arrayMap(x -> substring(data, amounts_values_idx + ((x - 1) * 64), 64), range(1, toInt32(amounts_length) + 1)) AS amounts_hex
177+
FROM batch_transfer_metadata
178+
)
179+
SELECT
180+
chain_id,
181+
token_address,
182+
from_address,
183+
to_address,
184+
token_type,
185+
block_number,
186+
block_timestamp,
187+
transaction_hash,
188+
log_index,
189+
sign,
190+
insert_timestamp,
191+
reinterpretAsUInt256(reverse(unhex(substring(hex_id, 1, 64)))) AS token_id,
192+
reinterpretAsUInt256(reverse(unhex(substring(hex_amount, 1, 64)))) AS amount
193+
FROM batch_transfer_logs
194+
ARRAY JOIN
195+
ids_hex AS hex_id,
196+
amounts_hex AS hex_amount

‎internal/tools/clickhouse_create_transactions_table.sql

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,24 @@ CREATE TABLE IF NOT EXISTS transactions (
3939
INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1,
4040
INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1,
4141
INDEX idx_function_selector function_selector TYPE bloom_filter GRANULARITY 1,
42+
PROJECTION txs_chainid_from_address
43+
(
44+
SELECT *
45+
ORDER BY
46+
chain_id,
47+
from_address,
48+
block_number
49+
),
50+
PROJECTION txs_chainid_to_address
51+
(
52+
SELECT *
53+
ORDER BY
54+
chain_id,
55+
to_address,
56+
block_number,
57+
hash
58+
)
4259
) ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
4360
ORDER BY (chain_id, block_number, hash)
44-
PARTITION BY chain_id;
61+
PARTITION BY chain_id
62+
SETTINGS deduplicate_merge_projection_mode = 'drop', lightweight_mutation_projection_mode = 'rebuild';

0 commit comments

Comments
 (0)