Skip to content

Commit 4db2943

Browse files
iamramtiniamramtin
committed
perf: optimize getAddressTransactions query to eliminate timeouts
- Restructure query with early canonical filtering and optimized CTEs - Add database indexes for correlated subqueries - Add optimized function and migration script Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com>
1 parent c9ef5e8 commit 4db2943

File tree

3 files changed

+354
-1
lines changed

3 files changed

+354
-1
lines changed

‎src/api/routes/v2/addresses.ts‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import {
33
handleCache,
44
handleChainTipCache,
55
handlePrincipalCache,
6-
handlePrincipalMempoolCache,
76
handleTransactionCache,
87
} from '../../../api/controllers/cache-controller';
98
import { AddressParamsSchema, AddressTransactionParamsSchema } from './schemas';
@@ -86,6 +85,59 @@ export const AddressRoutesV2: FastifyPluginAsync<
8685
}
8786
);
8887

88+
// Adding temporary dual endpoint for maintainer review and testing
89+
// eslint-disable-next-line no-warning-comments
90+
// TODO(iamramtin): Replace V1 after maintainer review and testing
91+
fastify.get(
92+
'/:address/transactions-v2',
93+
{
94+
preHandler: handlePrincipalCache,
95+
schema: {
96+
operationId: 'get_address_transactions_v2',
97+
summary: 'Get address transactions V2',
98+
description: `Retrieves a paginated list of confirmed transactions sent or received by a STX address or Smart Contract ID, alongside the total amount of STX sent or received and the number of STX, FT and NFT transfers contained within each transaction.
99+
100+
More information on Transaction types can be found [here](https://docs.stacks.co/understand-stacks/transactions#types).`,
101+
tags: ['Transactions'],
102+
params: AddressParamsSchema,
103+
querystring: Type.Object({
104+
limit: LimitParam(ResourceType.Tx),
105+
offset: OffsetParam(),
106+
exclude_function_args: ExcludeFunctionArgsParamSchema,
107+
}),
108+
response: {
109+
200: PaginatedResponse(AddressTransactionSchema),
110+
},
111+
},
112+
},
113+
async (req, reply) => {
114+
const params = req.params;
115+
const query = req.query;
116+
const excludeFunctionArgs = req.query.exclude_function_args ?? false;
117+
118+
try {
119+
const { limit, offset, results, total } = await fastify.db.v2.getAddressTransactionsV2({
120+
...params,
121+
...query,
122+
});
123+
const transfers: AddressTransaction[] = results.map(r =>
124+
parseDbTxWithAccountTransferSummary(r, excludeFunctionArgs)
125+
);
126+
await reply.send({
127+
limit,
128+
offset,
129+
total,
130+
results: transfers,
131+
});
132+
} catch (error) {
133+
if (error instanceof InvalidRequestError) {
134+
throw new NotFoundError(error.message);
135+
}
136+
throw error;
137+
}
138+
}
139+
);
140+
89141
fastify.get(
90142
'/:address/transactions/:tx_id/events',
91143
{

‎src/datastore/address-transactions-indexes.sql‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
-- =============================================================================
2+
-- MIGRATION SCRIPT: getAddressTransactions Optimization
3+
-- Eliminates query timeouts for high-transaction addresses
4+
-- =============================================================================
5+
6+
-- Verify no conflicting indexes exist (something like this)
7+
SELECT schemaname, tablename, indexname
8+
FROM pg_indexes
9+
WHERE indexname LIKE '%_canonical_optimized'
10+
OR indexname LIKE '%_subquery_optimized'
11+
ORDER BY tablename, indexname;
12+
13+
-- =============================================================================
14+
-- CANONICAL TRANSACTION FILTERING
15+
-- =============================================================================
16+
-- Problem: The query joins txs table for every transaction to check canonical = TRUE
17+
-- and microblock_canonical = TRUE. This creates expensive nested loops that scan
18+
-- thousands of transactions, applying filters after the join.
19+
--
20+
-- Solution: Create partial index containing only canonical transactions with
21+
-- built-in ordering. This eliminates the filter step entirely and supports
22+
-- efficient sorting without additional operations
23+
--
24+
-- Trade-off: Additional storage on txs table to get significant query speedup.
25+
-- But index only contains canonical transactions which should reduce overall size
26+
27+
CREATE INDEX CONCURRENTLY idx_txs_canonical_optimized
28+
ON txs (tx_id, index_block_hash, microblock_hash, block_height DESC, microblock_sequence DESC, tx_index DESC)
29+
WHERE canonical = TRUE AND microblock_canonical = TRUE;
30+
31+
-- Optional index `address_txs` CTE if it were materialized as its own table
32+
-- CREATE INDEX CONCURRENTLY idx_address_txs_dedupe
33+
-- ON address_txs (tx_id, index_block_hash, microblock_hash);
34+
35+
ANALYZE txs;
36+
37+
-- =============================================================================
38+
-- EVENT TABLE SUBQUERIES
39+
-- =============================================================================
40+
-- Problem: Each transaction requires 11 correlated subqueries that scan event tables
41+
-- using expensive bitmap operations. For 50 returned transactions, this means 550
42+
-- separate bitmap scans combining tx_id and index_block_hash lookups
43+
--
44+
-- Solution: Create compound indexes that cover all subquery conditions in a single
45+
-- lookup. The INCLUDE clause adds frequently accessed columns without increasing
46+
-- the index key size, enabling index-only scans
47+
--
48+
-- Trade-off: Additional storage per event table to remov bitmap
49+
-- operations and heap lookups in subqueries
50+
51+
-- STX Events: used in 5 subqueries per transaction
52+
CREATE INDEX CONCURRENTLY idx_stx_events_subquery_optimized
53+
ON stx_events (tx_id, index_block_hash, microblock_hash, asset_event_type_id)
54+
INCLUDE (amount, sender, recipient);
55+
56+
-- FT Events: used in 3 subqueries per transaction
57+
CREATE INDEX CONCURRENTLY idx_ft_events_subquery_optimized
58+
ON ft_events (tx_id, index_block_hash, microblock_hash, asset_event_type_id)
59+
INCLUDE (sender, recipient);
60+
61+
-- NFT Events: used in 3 subqueries
62+
CREATE INDEX CONCURRENTLY idx_nft_events_subquery_optimized
63+
ON nft_events (tx_id, index_block_hash, microblock_hash, asset_event_type_id)
64+
INCLUDE (sender, recipient);
65+
66+
ANALYZE stx_events, ft_events, nft_events;
67+
68+
-- =============================================================================
69+
-- MONITORING / VERIFICATION
70+
-- =============================================================================
71+
72+
-- Ensure all indexes were created successfully and are valid
73+
SELECT
74+
psi.schemaname,
75+
psi.relname as tablename,
76+
psi.indexrelname,
77+
pi.indisvalid as is_valid,
78+
pi.indisready as is_ready,
79+
pg_size_pretty(pg_relation_size(psi.indexrelid)) as index_size
80+
FROM pg_stat_user_indexes psi
81+
JOIN pg_index pi ON psi.indexrelid = pi.indexrelid
82+
WHERE psi.indexrelname LIKE '%_canonical_optimized'
83+
OR psi.indexrelname LIKE '%_subquery_optimized'
84+
ORDER BY psi.relname, psi.indexrelname;
85+
86+
-- Create view to monitor ongoing performance tracking
87+
CREATE OR REPLACE VIEW address_transactions_performance AS
88+
SELECT
89+
schemaname,
90+
relname as tablename,
91+
pg_stat_user_indexes.indexrelname,
92+
idx_scan as times_used,
93+
pg_size_pretty(pg_relation_size(indexrelid)) as index_size,
94+
CASE
95+
WHEN idx_scan = 0 THEN 'Not yet used'
96+
WHEN idx_scan < 100 THEN 'Low usage'
97+
ELSE 'Active'
98+
END as status
99+
FROM pg_stat_user_indexes
100+
WHERE pg_stat_user_indexes.indexrelname LIKE '%_canonical_optimized'
101+
OR pg_stat_user_indexes.indexrelname LIKE '%_subquery_optimized'
102+
ORDER BY idx_scan DESC;
103+
104+
SELECT * FROM address_transactions_performance;
105+
106+
-- Verify all indexes are valid and being used
107+
SELECT
108+
schemaname, relname, pg_stat_user_indexes.indexrelname, idx_scan,
109+
CASE
110+
WHEN idx_scan = 0 THEN 'INDEX NOT USED - INVESTIGATE'
111+
ELSE 'OK'
112+
END as health_status
113+
FROM pg_stat_user_indexes
114+
WHERE pg_stat_user_indexes.indexrelname LIKE '%_optimized'
115+
ORDER BY idx_scan DESC;
116+
117+
/*
118+
-- Rollback:
119+
DROP INDEX CONCURRENTLY IF EXISTS idx_stx_events_subquery_optimized;
120+
DROP INDEX CONCURRENTLY IF EXISTS idx_ft_events_subquery_optimized;
121+
DROP INDEX CONCURRENTLY IF EXISTS idx_nft_events_subquery_optimized;
122+
DROP INDEX CONCURRENTLY IF EXISTS idx_txs_canonical_optimized;
123+
124+
ANALYZE txs, stx_events, ft_events, nft_events;
125+
126+
DROP VIEW IF EXISTS address_transactions_performance;
127+
*/

‎src/datastore/pg-store-v2.ts‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,7 @@ export class PgStoreV2 extends BasePgStoreModule {
526526
});
527527
}
528528

529+
// Original implementation - kept for comparison during optimization review
529530
async getAddressTransactions(
530531
args: AddressParams & TransactionPaginationQueryParams
531532
): Promise<DbPaginatedResult<DbTxWithAddressTransfers>> {
@@ -645,6 +646,179 @@ export class PgStoreV2 extends BasePgStoreModule {
645646
});
646647
}
647648

649+
// Optimized implementation
650+
// Key optimizations: early canonical filtering, efficient HashAggregate counting, parallel execution
651+
// eslint-disable-next-line no-warning-comments
652+
// TODO(iamramtin): Replace V1 after maintainer review and testing
653+
async getAddressTransactionsV2(
654+
args: AddressParams & TransactionPaginationQueryParams
655+
): Promise<DbPaginatedResult<DbTxWithAddressTransfers>> {
656+
return await this.sqlTransaction(async sql => {
657+
const limit = args.limit ?? TransactionLimitParamSchema.default;
658+
const offset = args.offset ?? 0;
659+
660+
const eventCond = sql`
661+
tx_id = limited_txs.tx_id
662+
AND index_block_hash = limited_txs.index_block_hash
663+
AND microblock_hash = limited_txs.microblock_hash
664+
`;
665+
666+
const eventAcctCond = sql`
667+
${eventCond} AND (sender = ${args.address} OR recipient = ${args.address})
668+
`;
669+
670+
const resultQuery = await sql<(AddressTransfersTxQueryResult & { count: number })[]>`
671+
WITH address_txs AS (
672+
(
673+
SELECT
674+
t.tx_id,
675+
t.index_block_hash,
676+
t.microblock_hash,
677+
t.block_height AS sort_block_height,
678+
t.microblock_sequence AS sort_microblock_sequence,
679+
t.tx_index AS sort_tx_index
680+
FROM principal_stx_txs p
681+
INNER JOIN txs t USING (tx_id, index_block_hash, microblock_hash)
682+
WHERE p.principal = ${args.address}
683+
AND t.canonical = TRUE
684+
AND t.microblock_canonical = TRUE
685+
)
686+
UNION ALL
687+
(
688+
SELECT
689+
t.tx_id,
690+
t.index_block_hash,
691+
t.microblock_hash,
692+
t.block_height AS sort_block_height,
693+
t.microblock_sequence AS sort_microblock_sequence,
694+
t.tx_index AS sort_tx_index
695+
FROM stx_events se
696+
INNER JOIN txs t USING (tx_id, index_block_hash, microblock_hash)
697+
WHERE (se.sender = ${args.address} OR se.recipient = ${args.address})
698+
AND t.canonical = TRUE
699+
AND t.microblock_canonical = TRUE
700+
)
701+
UNION ALL
702+
(
703+
SELECT
704+
t.tx_id,
705+
t.index_block_hash,
706+
t.microblock_hash,
707+
t.block_height AS sort_block_height,
708+
t.microblock_sequence AS sort_microblock_sequence,
709+
t.tx_index AS sort_tx_index
710+
FROM ft_events fe
711+
INNER JOIN txs t USING (tx_id, index_block_hash, microblock_hash)
712+
WHERE (fe.sender = ${args.address} OR fe.recipient = ${args.address})
713+
AND t.canonical = TRUE
714+
AND t.microblock_canonical = TRUE
715+
)
716+
UNION ALL
717+
(
718+
SELECT
719+
t.tx_id,
720+
t.index_block_hash,
721+
t.microblock_hash,
722+
t.block_height AS sort_block_height,
723+
t.microblock_sequence AS sort_microblock_sequence,
724+
t.tx_index AS sort_tx_index
725+
FROM nft_events ne
726+
INNER JOIN txs t USING (tx_id, index_block_hash, microblock_hash)
727+
WHERE (ne.sender = ${args.address} OR ne.recipient = ${args.address})
728+
AND t.canonical = TRUE
729+
AND t.microblock_canonical = TRUE
730+
)
731+
),
732+
deduped_txs AS (
733+
SELECT DISTINCT
734+
tx_id,
735+
index_block_hash,
736+
microblock_hash,
737+
sort_block_height,
738+
sort_microblock_sequence,
739+
sort_tx_index
740+
FROM address_txs
741+
),
742+
limited_txs AS (
743+
SELECT *
744+
FROM deduped_txs
745+
ORDER BY sort_block_height DESC, sort_microblock_sequence DESC, sort_tx_index DESC
746+
LIMIT ${limit}
747+
OFFSET ${offset}
748+
)
749+
SELECT
750+
${sql(TX_COLUMNS)},
751+
(
752+
SELECT COALESCE(SUM(amount), 0)
753+
FROM stx_events
754+
WHERE ${eventCond} AND sender = ${args.address}
755+
) +
756+
CASE
757+
WHEN (txs.sponsored = false AND txs.sender_address = ${args.address})
758+
OR (txs.sponsored = true AND txs.sponsor_address = ${args.address})
759+
THEN txs.fee_rate ELSE 0
760+
END AS stx_sent,
761+
(
762+
SELECT COALESCE(SUM(amount), 0)
763+
FROM stx_events
764+
WHERE ${eventCond} AND recipient = ${args.address}
765+
) AS stx_received,
766+
(
767+
SELECT COUNT(*)::int FROM stx_events
768+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Transfer}
769+
) AS stx_transfer,
770+
(
771+
SELECT COUNT(*)::int FROM stx_events
772+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Mint}
773+
) AS stx_mint,
774+
(
775+
SELECT COUNT(*)::int FROM stx_events
776+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Burn}
777+
) AS stx_burn,
778+
(
779+
SELECT COUNT(*)::int FROM ft_events
780+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Transfer}
781+
) AS ft_transfer,
782+
(
783+
SELECT COUNT(*)::int FROM ft_events
784+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Mint}
785+
) AS ft_mint,
786+
(
787+
SELECT COUNT(*)::int FROM ft_events
788+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Burn}
789+
) AS ft_burn,
790+
(
791+
SELECT COUNT(*)::int FROM nft_events
792+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Transfer}
793+
) AS nft_transfer,
794+
(
795+
SELECT COUNT(*)::int FROM nft_events
796+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Mint}
797+
) AS nft_mint,
798+
(
799+
SELECT COUNT(*)::int FROM nft_events
800+
WHERE ${eventAcctCond} AND asset_event_type_id = ${DbAssetEventTypeId.Burn}
801+
) AS nft_burn,
802+
(
803+
SELECT COUNT(*)::int FROM deduped_txs
804+
) AS count
805+
FROM limited_txs
806+
INNER JOIN txs USING (tx_id, index_block_hash, microblock_hash)
807+
ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC
808+
`;
809+
810+
const total = resultQuery.length > 0 ? resultQuery[0].count : 0;
811+
const parsed = resultQuery.map(r => parseAccountTransferSummaryTxQueryResult(r));
812+
813+
return {
814+
total,
815+
limit,
816+
offset,
817+
results: parsed,
818+
};
819+
});
820+
}
821+
648822
async getAddressTransactionEvents(args: {
649823
limit: number;
650824
offset: number;

0 commit comments

Comments
 (0)