From aefbdb5e7bc571376392be41cce9ef697c29d1c8 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 27 Oct 2025 18:07:27 -0400 Subject: [PATCH 1/5] New Vault Pnl tables --- .../__tests__/stores/vault-pnl-view.test.ts | 204 ++++++++++++++++++ ...51027172540_create_vaults_hourly_pnl_v2.ts | 47 ++++ ...251027172547_create_vaults_daily_pnl_v2.ts | 47 ++++ .../packages/postgres/src/stores/pnl-table.ts | 10 + .../postgres/src/stores/vault-pnl-view.ts | 99 +++++++++ .../packages/postgres/src/types/pnl-types.ts | 5 + 6 files changed, 412 insertions(+) create mode 100644 indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts create mode 100644 indexer/packages/postgres/src/stores/vault-pnl-view.ts diff --git a/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts b/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts new file mode 100644 index 00000000000..e25bb7f9d57 --- /dev/null +++ b/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts @@ -0,0 +1,204 @@ +import { DateTime } from 'luxon'; +import { PnlInterval, PnlFromDatabase } from '../../src/types'; +import * as VaultPnlView from '../../src/stores/vault-pnl-view'; +import * as PnlTable from '../../src/stores/pnl-table'; +import * as VaultTable from '../../src/stores/vault-table'; +import * as WalletTable from '../../src/stores/wallet-table'; +import * as SubaccountTable from '../../src/stores/subaccount-table'; +import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; +import { seedData } from '../helpers/mock-generators'; +import { + defaultSubaccountId, + defaultSubaccountIdWithAlternateAddress, + defaultSubaccountWithAlternateAddress, + defaultWallet2, + defaultVault, + defaultSubaccount, +} from '../helpers/constants'; + +describe('VaultPnl store', () => { + beforeAll(async () => { + await migrate(); + }); + + beforeEach(async () => { + await seedData(); + await WalletTable.create(defaultWallet2); + await SubaccountTable.create(defaultSubaccountWithAlternateAddress); + await Promise.all([ + VaultTable.create({ + ...defaultVault, + address: defaultSubaccount.address, + }), + VaultTable.create({ + ...defaultVault, + address: defaultSubaccountWithAlternateAddress.address, + }), + ]); + }); + + afterEach(async () => { + await clearData(); + }); + + afterAll(async () => { + await teardown(); + }); + + it.each([ + { + description: 'Get hourly vault pnl', + interval: PnlInterval.hour, + }, + { + description: 'Get daily vault pnl', + interval: PnlInterval.day, + }, + ])('$description', async ({ + interval, + }: { + interval: PnlInterval, + }) => { + const createdPnl: PnlFromDatabase[] = await setupIntervalPnl(); + + await VaultPnlView.refreshDailyView(); + await VaultPnlView.refreshHourlyView(); + + const pnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + interval, + 7 * 24 * 60 * 60, // 1 week + DateTime.fromISO(createdPnl[8].createdAt).plus({ seconds: 1 }), + ); + + // See setup function for created pnl records. + // Should exclude records that are within the same hour except the first. + const expectedHourlyPnl: PnlFromDatabase[] = [ + createdPnl[7], + createdPnl[5], + createdPnl[2], + createdPnl[0], + ]; + + // Should exclude records that are within the same day except for the first. + const expectedDailyPnl: PnlFromDatabase[] = [ + createdPnl[7], + createdPnl[2], + ]; + + if (interval === PnlInterval.day) { + expect(pnlData).toEqual(expectedDailyPnl); + } else if (interval === PnlInterval.hour) { + expect(pnlData).toEqual(expectedHourlyPnl); + } + }); + + it('Get latest vault pnl', async () => { + await setupIntervalPnl(); + + await VaultPnlView.refreshHourlyView(); + + const latestPnl: PnlFromDatabase[] = await VaultPnlView.getLatestVaultPnl(); + + // Should return the most recent PNL for each subaccount + expect(latestPnl).toHaveLength(2); + + const subaccount1Pnl = latestPnl.find( + (pnl) => pnl.subaccountId === defaultSubaccountId, + ); + const subaccount2Pnl = latestPnl.find( + (pnl) => pnl.subaccountId === defaultSubaccountIdWithAlternateAddress, + ); + + expect(subaccount1Pnl).toBeDefined(); + expect(subaccount1Pnl?.equity).toBe('1100'); + + expect(subaccount2Pnl).toBeDefined(); + expect(subaccount2Pnl?.equity).toBe('200'); + }); + + async function setupIntervalPnl(): Promise { + const currentTime: DateTime = DateTime.utc().startOf('day'); + const tenMinAgo: string = currentTime.minus({ minute: 10 }).toISO(); + const almostTenMinAgo: string = currentTime.minus({ second: 603 }).toISO(); + const twoHoursAgo: string = currentTime.minus({ hour: 2 }).toISO(); + const twoDaysAgo: string = currentTime.minus({ day: 2 }).toISO(); + const monthAgo: string = currentTime.minus({ day: 30 }).toISO(); + + const createdPnl: PnlFromDatabase[] = await PnlTable.createMany([ + { + subaccountId: defaultSubaccountId, + equity: '1100', + createdAt: almostTenMinAgo, + createdAtHeight: '10', + totalPnl: '1200', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1090', + createdAt: tenMinAgo, + createdAtHeight: '8', + totalPnl: '1190', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1080', + createdAt: twoHoursAgo, + createdAtHeight: '6', + totalPnl: '1180', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1070', + createdAt: twoDaysAgo, + createdAtHeight: '4', + totalPnl: '1170', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1200', + createdAt: monthAgo, + createdAtHeight: '3', + totalPnl: '1170', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '200', + createdAt: almostTenMinAgo, + createdAtHeight: '10', + totalPnl: '300', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '210', + createdAt: tenMinAgo, + createdAtHeight: '8', + totalPnl: '310', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '220', + createdAt: twoHoursAgo, + createdAtHeight: '6', + totalPnl: '320', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '230', + createdAt: twoDaysAgo, + createdAtHeight: '4', + totalPnl: '330', + netTransfers: '50', + }, + ]); + + return createdPnl; + } +}); \ No newline at end of file diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts new file mode 100644 index 00000000000..6b14bbee20a --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts @@ -0,0 +1,47 @@ +import * as Knex from 'knex'; + +const RAW_VAULTS_PNL_HOURLY_V2_QUERY: string = ` +CREATE MATERIALIZED VIEW IF NOT EXISTS vaults_hourly_pnl_v2 AS +WITH vault_subaccounts AS ( + SELECT subaccounts.id + FROM vaults, subaccounts + WHERE vaults.address = subaccounts.address + AND subaccounts."subaccountNumber" = 0 +), +pnl_subaccounts AS ( + SELECT * FROM vault_subaccounts + UNION + SELECT id FROM subaccounts + WHERE address = 'dydx18tkxrnrkqc2t0lr3zxr5g6a4hdvqksylxqje4r' + AND "subaccountNumber" = 0 +) +SELECT + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" +FROM ( + SELECT + pnl.*, + ROW_NUMBER() OVER ( + partition BY "subaccountId", DATE_TRUNC('hour', "createdAt") + ORDER BY "createdAt" + ) AS r + FROM pnl + WHERE "subaccountId" IN (SELECT * FROM pnl_subaccounts) + AND "createdAt" >= NOW() - interval '604800 second' +) AS pnl_intervals +WHERE r = 1 +ORDER BY "subaccountId"; +`; + +export async function up(knex: Knex): Promise { + await knex.raw(RAW_VAULTS_PNL_HOURLY_V2_QUERY); + await knex.raw('CREATE UNIQUE INDEX ON vaults_hourly_pnl_v2 ("subaccountId", "createdAt");'); +} + +export async function down(knex: Knex): Promise { + await knex.raw('DROP MATERIALIZED VIEW IF EXISTS vaults_hourly_pnl_v2;'); +} diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts new file mode 100644 index 00000000000..4c0a64279c9 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts @@ -0,0 +1,47 @@ +import * as Knex from 'knex'; + +const RAW_VAULTS_PNL_DAILY_V2_QUERY: string = ` +CREATE MATERIALIZED VIEW IF NOT EXISTS vaults_daily_pnl_v2 AS +WITH vault_subaccounts AS ( + SELECT subaccounts.id + FROM vaults, subaccounts + WHERE vaults.address = subaccounts.address + AND subaccounts."subaccountNumber" = 0 +), +pnl_subaccounts AS ( + SELECT * FROM vault_subaccounts + UNION + SELECT id FROM subaccounts + WHERE address = 'dydx18tkxrnrkqc2t0lr3zxr5g6a4hdvqksylxqje4r' + AND "subaccountNumber" = 0 +) +SELECT + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" +FROM ( + SELECT + pnl.*, + ROW_NUMBER() OVER ( + partition BY "subaccountId", DATE_TRUNC('day', "createdAt") + ORDER BY "createdAt" + ) AS r + FROM pnl + WHERE "subaccountId" IN (SELECT * FROM pnl_subaccounts) + AND "createdAt" >= NOW() - interval '7776000 second' +) AS pnl_intervals +WHERE r = 1 +ORDER BY "subaccountId"; +`; + +export async function up(knex: Knex): Promise { + await knex.raw(RAW_VAULTS_PNL_DAILY_V2_QUERY); + await knex.raw('CREATE UNIQUE INDEX ON vaults_daily_pnl_v2 ("subaccountId", "createdAt");'); +} + +export async function down(knex: Knex): Promise { + await knex.raw('DROP MATERIALIZED VIEW IF EXISTS vaults_daily_pnl_v2;'); +} diff --git a/indexer/packages/postgres/src/stores/pnl-table.ts b/indexer/packages/postgres/src/stores/pnl-table.ts index d8a211fa52c..8eaeb5d0aa1 100644 --- a/indexer/packages/postgres/src/stores/pnl-table.ts +++ b/indexer/packages/postgres/src/stores/pnl-table.ts @@ -373,6 +373,16 @@ export async function create( ).insert(pnlToCreate).returning('*'); } +export async function createMany( + pnls: PnlCreateObject[], + options: Options = { txId: undefined }, +): Promise { + return PnlModel + .query(Transaction.get(options.txId)) + .insert(pnls) + .returning('*'); +} + export async function findById( subaccountId: string, createdAt: string, diff --git a/indexer/packages/postgres/src/stores/vault-pnl-view.ts b/indexer/packages/postgres/src/stores/vault-pnl-view.ts new file mode 100644 index 00000000000..e5a46d1c3f1 --- /dev/null +++ b/indexer/packages/postgres/src/stores/vault-pnl-view.ts @@ -0,0 +1,99 @@ +import { DateTime } from 'luxon'; +import { knexReadReplica } from '../helpers/knex'; +import { rawQuery } from '../helpers/stores-helpers'; +import { PnlFromDatabase, PnlInterval } from '../types'; + +const VAULT_HOURLY_PNL_VIEW: string = 'vaults_hourly_pnl_v2'; +const VAULT_DAILY_PNL_VIEW: string = 'vaults_daily_pnl_v2'; + +/** + * Refresh the hourly vault PNL materialized view. + */ +export async function refreshHourlyView(): Promise { + await rawQuery( + `REFRESH MATERIALIZED VIEW CONCURRENTLY ${VAULT_HOURLY_PNL_VIEW}`, + { + readReplica: false, + }, + ); +} + +/** + * Refresh the daily vault PNL materialized view. + */ +export async function refreshDailyView(): Promise { + await rawQuery( + `REFRESH MATERIALIZED VIEW CONCURRENTLY ${VAULT_DAILY_PNL_VIEW}`, + { + readReplica: false, + }, + ); +} + +/** + * Get vault PNL data for a given interval and time window. + * + * @param interval - The PNL tick interval (hour or day) + * @param timeWindowSeconds - The time window in seconds + * @param earliestDate - The earliest date to fetch data from + * @returns Array of vault PNL records + */ +export async function getVaultsPnl( + interval: PnlInterval, + timeWindowSeconds: number, + earliestDate: DateTime, +): Promise { + const viewName: string = interval === PnlInterval.hour + ? VAULT_HOURLY_PNL_VIEW + : VAULT_DAILY_PNL_VIEW; + + const result: { + rows: PnlFromDatabase[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" + FROM ${viewName} + WHERE + "createdAt" >= '${earliestDate.toUTC().toISO()}'::timestamp AND + "createdAt" > NOW() - INTERVAL '${timeWindowSeconds} second' + ORDER BY "subaccountId", "createdAt"; + `, + ) as unknown as { + rows: PnlFromDatabase[], + }; + + return result.rows; +} + +/** + * Get the latest vault PNL snapshot for each vault. + * + * @returns Array of latest vault PNL records, one per vault + */ +export async function getLatestVaultPnl(): Promise { + const result: { + rows: PnlFromDatabase[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT DISTINCT ON ("subaccountId") + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" + FROM ${VAULT_HOURLY_PNL_VIEW} + ORDER BY "subaccountId", "createdAt" DESC; + `, + ) as unknown as { + rows: PnlFromDatabase[], + }; + + return result.rows; +} \ No newline at end of file diff --git a/indexer/packages/postgres/src/types/pnl-types.ts b/indexer/packages/postgres/src/types/pnl-types.ts index eade4df2f17..8cf789ff752 100644 --- a/indexer/packages/postgres/src/types/pnl-types.ts +++ b/indexer/packages/postgres/src/types/pnl-types.ts @@ -15,3 +15,8 @@ export interface PnlCreateObject { netTransfers: string, totalPnl: string, } + +export enum PnlInterval { + hour = 'hour', + day = 'day', +} From 641537044e35621694d0c121f5fa71adb0df9ec0 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 28 Oct 2025 16:25:02 -0400 Subject: [PATCH 2/5] Update refresh vault pnl task to refresh old and new tables --- .../__tests__/stores/vault-pnl-view.test.ts | 6 +- indexer/packages/postgres/src/index.ts | 1 + .../packages/postgres/src/stores/pnl-table.ts | 9 +- .../postgres/src/stores/vault-pnl-view.ts | 24 ++- .../__tests__/tasks/refresh-vault-pnl.test.ts | 163 +++++++++++++----- .../roundtable/src/tasks/refresh-vault-pnl.ts | 57 +++++- 6 files changed, 196 insertions(+), 64 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts b/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts index e25bb7f9d57..7f1de7eab99 100644 --- a/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts +++ b/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts @@ -101,7 +101,7 @@ describe('VaultPnl store', () => { // Should return the most recent PNL for each subaccount expect(latestPnl).toHaveLength(2); - + const subaccount1Pnl = latestPnl.find( (pnl) => pnl.subaccountId === defaultSubaccountId, ); @@ -111,7 +111,7 @@ describe('VaultPnl store', () => { expect(subaccount1Pnl).toBeDefined(); expect(subaccount1Pnl?.equity).toBe('1100'); - + expect(subaccount2Pnl).toBeDefined(); expect(subaccount2Pnl?.equity).toBe('200'); }); @@ -201,4 +201,4 @@ describe('VaultPnl store', () => { return createdPnl; } -}); \ No newline at end of file +}); diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts index 1efadbbfe9d..f2f980e536e 100644 --- a/indexer/packages/postgres/src/index.ts +++ b/indexer/packages/postgres/src/index.ts @@ -56,6 +56,7 @@ export * as VaultTable from './stores/vault-table'; export * as FundingPaymentsTable from './stores/funding-payments-table'; export * as TurnkeyUsersTable from './stores/turnkey-users-table'; export * as VaultPnlTicksView from './stores/vault-pnl-ticks-view'; +export * as VaultPnlView from './stores/vault-pnl-view'; export * as PermissionApprovalTable from './stores/permission-approval-table'; export * as PnlTable from './stores/pnl-table'; export * as BridgeInformationTable from './stores/bridge-information-table'; diff --git a/indexer/packages/postgres/src/stores/pnl-table.ts b/indexer/packages/postgres/src/stores/pnl-table.ts index 8eaeb5d0aa1..0a3d974370e 100644 --- a/indexer/packages/postgres/src/stores/pnl-table.ts +++ b/indexer/packages/postgres/src/stores/pnl-table.ts @@ -377,10 +377,11 @@ export async function createMany( pnls: PnlCreateObject[], options: Options = { txId: undefined }, ): Promise { - return PnlModel - .query(Transaction.get(options.txId)) - .insert(pnls) - .returning('*'); + if (!Array.isArray(pnls) || pnls.length === 0) { + return []; + } + const qb = PnlModel.query(Transaction.get(options.txId)); + return qb.insert(pnls).returning('*'); } export async function findById( diff --git a/indexer/packages/postgres/src/stores/vault-pnl-view.ts b/indexer/packages/postgres/src/stores/vault-pnl-view.ts index e5a46d1c3f1..5bf24725f7c 100644 --- a/indexer/packages/postgres/src/stores/vault-pnl-view.ts +++ b/indexer/packages/postgres/src/stores/vault-pnl-view.ts @@ -1,4 +1,5 @@ import { DateTime } from 'luxon'; + import { knexReadReplica } from '../helpers/knex'; import { rawQuery } from '../helpers/stores-helpers'; import { PnlFromDatabase, PnlInterval } from '../types'; @@ -32,7 +33,7 @@ export async function refreshDailyView(): Promise { /** * Get vault PNL data for a given interval and time window. - * + * * @param interval - The PNL tick interval (hour or day) * @param timeWindowSeconds - The time window in seconds * @param earliestDate - The earliest date to fetch data from @@ -43,9 +44,15 @@ export async function getVaultsPnl( timeWindowSeconds: number, earliestDate: DateTime, ): Promise { - const viewName: string = interval === PnlInterval.hour - ? VAULT_HOURLY_PNL_VIEW - : VAULT_DAILY_PNL_VIEW; + const VIEW_BY_INTERVAL: Record = { + [PnlInterval.hour]: VAULT_HOURLY_PNL_VIEW, + [PnlInterval.day]: VAULT_DAILY_PNL_VIEW, + }; + const viewName = VIEW_BY_INTERVAL[interval]; + if (!Number.isFinite(timeWindowSeconds) || timeWindowSeconds <= 0) { + throw new Error('timeWindowSeconds must be a positive number'); + } + const earliest = earliestDate.toUTC().toJSDate(); // lets pg type it as timestamptz const result: { rows: PnlFromDatabase[], @@ -60,10 +67,11 @@ export async function getVaultsPnl( "createdAtHeight" FROM ${viewName} WHERE - "createdAt" >= '${earliestDate.toUTC().toISO()}'::timestamp AND - "createdAt" > NOW() - INTERVAL '${timeWindowSeconds} second' + "createdAt" >= ? AND + "createdAt" > NOW() - make_interval(secs => ?) ORDER BY "subaccountId", "createdAt"; `, + [earliest, Math.trunc(timeWindowSeconds)], ) as unknown as { rows: PnlFromDatabase[], }; @@ -73,7 +81,7 @@ export async function getVaultsPnl( /** * Get the latest vault PNL snapshot for each vault. - * + * * @returns Array of latest vault PNL records, one per vault */ export async function getLatestVaultPnl(): Promise { @@ -96,4 +104,4 @@ export async function getLatestVaultPnl(): Promise { }; return result.rows; -} \ No newline at end of file +} diff --git a/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts b/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts index 0078dbb74ae..aa72e2e66ca 100644 --- a/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts @@ -1,12 +1,16 @@ import config from '../../src/config'; -import refreshVaulPnlTask from '../../src/tasks/refresh-vault-pnl'; +import refreshVaultPnlTask from '../../src/tasks/refresh-vault-pnl'; import { Settings, DateTime } from 'luxon'; import { BlockTable, + PnlFromDatabase, + PnlInterval, + PnlTable, PnlTickInterval, PnlTicksFromDatabase, PnlTicksTable, VaultPnlTicksView, + VaultPnlView, VaultTable, dbHelpers, testConstants, @@ -40,91 +44,166 @@ describe('refresh-vault-pnl', () => { afterEach(async () => { await dbHelpers.clearData(); - await VaultPnlTicksView.refreshDailyView(); - await VaultPnlTicksView.refreshHourlyView(); + // Refresh both old and new views + await Promise.all([ + VaultPnlTicksView.refreshDailyView(), + VaultPnlTicksView.refreshHourlyView(), + VaultPnlView.refreshDailyView(), + VaultPnlView.refreshHourlyView(), + ]); jest.clearAllMocks(); Settings.now = () => new Date().valueOf(); }); - it('refreshes hourly view if within time window of an hour', async () => { + it('refreshes hourly views (old and new) if within time window of an hour', async () => { Settings.now = () => currentTime.startOf('hour').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS - 1 }, ).valueOf(); - const pnlTick: PnlTicksFromDatabase = await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Verify old view was refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.hour, 86400, currentTime.minus({ day: 1 }), ); - expect(pnlTicks).toEqual([pnlTick]); + + // Verify new view was refreshed + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.hour, + 86400, + currentTime.minus({ day: 1 }), + ); + + // Both views should have the same data + expect(oldPnlTicks).toHaveLength(1); + expect(newPnlData).toHaveLength(1); + + // Verify data matches + expect(oldPnlTicks[0].subaccountId).toEqual(newPnlData[0].subaccountId); + expect(oldPnlTicks[0].equity).toEqual(newPnlData[0].equity); + expect(oldPnlTicks[0].totalPnl).toEqual(newPnlData[0].totalPnl); + expect(oldPnlTicks[0].netTransfers).toEqual(newPnlData[0].netTransfers); }); - it('refreshes daily view if within time window of a day', async () => { + it('refreshes daily views (old and new) if within time window of a day', async () => { Settings.now = () => currentTime.startOf('day').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS - 1 }, ).valueOf(); - const pnlTick: PnlTicksFromDatabase = await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Verify old view was refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.day, 608400, currentTime.minus({ day: 7 }), ); - expect(pnlTicks).toEqual([pnlTick]); + + // Verify new view was refreshed + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.day, + 608400, + currentTime.minus({ day: 7 }), + ); + + // Both views should have the same data + expect(oldPnlTicks).toHaveLength(1); + expect(newPnlData).toHaveLength(1); + + expect(oldPnlTicks[0].subaccountId).toEqual(newPnlData[0].subaccountId); + expect(newPnlData[0].subaccountId).toEqual(testConstants.defaultSubaccountId); }); - it('does not refresh hourly view if outside of time window of an hour', async () => { + it('does not refresh hourly views if outside of time window of an hour', async () => { Settings.now = () => currentTime.startOf('hour').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS + 1 }, ).valueOf(); - await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Neither view should have been refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.hour, 86400, currentTime.minus({ day: 1 }), ); - expect(pnlTicks).toEqual([]); + + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.hour, + 86400, + currentTime.minus({ day: 1 }), + ); + + expect(oldPnlTicks).toEqual([]); + expect(newPnlData).toEqual([]); }); - it('does not refresh daily view if outside time window of a day', async () => { + it('does not refresh daily views if outside time window of a day', async () => { Settings.now = () => currentTime.startOf('day').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS + 1 }, ).valueOf(); - await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Neither view should have been refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.day, 608400, currentTime.minus({ day: 7 }), ); - expect(pnlTicks).toEqual([]); + + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.day, + 608400, + currentTime.minus({ day: 7 }), + ); + + expect(oldPnlTicks).toEqual([]); + expect(newPnlData).toEqual([]); }); - async function setupPnlTick(): Promise { + /** + * Setup PNL data in both old (pnl_ticks) and new (pnl) tables + * to test that both views refresh correctly during migration period. + */ + async function setupPnlData(): Promise { const twoHoursAgo: string = currentTime.minus({ hour: 2 }).toISO(); - await Promise.all([ - BlockTable.create({ - blockHeight: '6', - time: twoHoursAgo, - }), - ]); - const createdTick: PnlTicksFromDatabase = await PnlTicksTable.create( - { - subaccountId: testConstants.defaultSubaccountId, - equity: '1080', - createdAt: twoHoursAgo, - totalPnl: '1180', - netTransfers: '50', - blockHeight: '6', - blockTime: twoHoursAgo, - }, - ); - return createdTick; + + // Create block for old pnl_ticks table (has foreign key to blocks) + await BlockTable.create({ + blockHeight: '6', + time: twoHoursAgo, + }); + + // Create data in old pnl_ticks table + await PnlTicksTable.create({ + subaccountId: testConstants.defaultSubaccountId, + equity: '1080', + createdAt: twoHoursAgo, + totalPnl: '1180', + netTransfers: '50', + blockHeight: '6', + blockTime: twoHoursAgo, + }); + + // Create data in new pnl table + await PnlTable.create({ + subaccountId: testConstants.defaultSubaccountId, + equity: '1080', + createdAt: twoHoursAgo, + createdAtHeight: '6', + totalPnl: '1180', + netTransfers: '50', + }); } }); diff --git a/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts b/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts index 635ef5526f5..aba50e4fd92 100644 --- a/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts +++ b/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts @@ -1,6 +1,7 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; import { VaultPnlTicksView, + VaultPnlView, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; @@ -13,36 +14,78 @@ export default async function runTask(): Promise { const taskStart: number = Date.now(); try { const currentTime: DateTime = DateTime.utc(); + + // Refresh hourly views if (currentTime.diff( currentTime.startOf('hour'), ).toMillis() < config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS) { logger.info({ at: 'refresh-vault-pnl#runTask', - message: 'Refreshing vault hourly pnl view', + message: 'Refreshing vault hourly pnl views (old and new)', currentTime, }); - await VaultPnlTicksView.refreshHourlyView(); + + const hourlyStart: number = Date.now(); + + // Refresh both old and new views in parallel + await Promise.all([ + VaultPnlTicksView.refreshHourlyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed old hourly view (vaults_hourly_pnl)', + }); + }), + VaultPnlView.refreshHourlyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed new hourly view (vaults_hourly_pnl_v2)', + }); + }), + ]); + stats.timing( `${config.SERVICE_NAME}.refresh-vault-pnl.hourly-view.timing`, - Date.now() - taskStart, + Date.now() - hourlyStart, ); } + // Refresh daily views if (currentTime.diff( currentTime.startOf('day'), ).toMillis() < config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS) { - const refreshDailyStart: number = Date.now(); logger.info({ at: 'refresh-vault-pnl#runTask', - message: 'Refreshing vault daily pnl view', + message: 'Refreshing vault daily pnl views (old and new)', currentTime, }); - await VaultPnlTicksView.refreshDailyView(); + + const dailyStart: number = Date.now(); + + // Refresh both old and new views in parallel + await Promise.all([ + VaultPnlTicksView.refreshDailyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed old daily view (vaults_daily_pnl)', + }); + }), + VaultPnlView.refreshDailyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed new daily view (vaults_daily_pnl_v2)', + }); + }), + ]); + stats.timing( `${config.SERVICE_NAME}.refresh-vault-pnl.daily-view.timing`, - Date.now() - refreshDailyStart, + Date.now() - dailyStart, ); } + stats.timing( + `${config.SERVICE_NAME}.refresh-vault-pnl.total.timing`, + Date.now() - taskStart, + ); } catch (error) { logger.error({ at: 'refresh-vault-pnl#runTask', From d0f0a52b6768969dfdbaa026b1cbe9db1c43b603 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 28 Oct 2025 17:00:57 -0400 Subject: [PATCH 3/5] Add pnl to new athena table --- .../src/lib/athena-ddl-tables/pnl.ts | 43 +++++++++++++++++++ .../src/tasks/update-research-environment.ts | 2 + 2 files changed, 45 insertions(+) create mode 100644 indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts diff --git a/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts b/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts new file mode 100644 index 00000000000..e279e61f4c3 --- /dev/null +++ b/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts @@ -0,0 +1,43 @@ +import { + getAthenaTableCreationStatement, + getExternalAthenaTableCreationStatement, + castToDouble, + castToTimestamp, +} from '../../helpers/sql'; + +const TABLE_NAME: string = 'pnl'; + +const RAW_TABLE_COLUMNS: string = ` + \`subaccountId\` string, + \`equity\` string, + \`totalPnl\` string, + \`netTransfers\` string, + \`createdAt\` string, + \`createdAtHeight\` bigint +`; + +const TABLE_COLUMNS: string = ` + "subaccountId", + ${castToDouble('equity')}, + ${castToDouble('totalPnl')}, + ${castToDouble('netTransfers')}, + ${castToTimestamp('createdAt')}, + "createdAtHeight" +`; + +export function generateRawTable(tablePrefix: string, rdsExportIdentifier: string): string { + return getExternalAthenaTableCreationStatement( + tablePrefix, + rdsExportIdentifier, + TABLE_NAME, + RAW_TABLE_COLUMNS, + ); +} + +export function generateTable(tablePrefix: string): string { + return getAthenaTableCreationStatement( + tablePrefix, + TABLE_NAME, + TABLE_COLUMNS, + ); +} \ No newline at end of file diff --git a/indexer/services/roundtable/src/tasks/update-research-environment.ts b/indexer/services/roundtable/src/tasks/update-research-environment.ts index 6a578dcdec4..929974d5f4d 100644 --- a/indexer/services/roundtable/src/tasks/update-research-environment.ts +++ b/indexer/services/roundtable/src/tasks/update-research-environment.ts @@ -33,6 +33,7 @@ import * as athenaOrders from '../lib/athena-ddl-tables/orders'; import * as athenaPerpetualMarkets from '../lib/athena-ddl-tables/perpetual_markets'; import * as athenaPerpetualPositions from '../lib/athena-ddl-tables/perpetual_positions'; import * as athenaPnlTicks from '../lib/athena-ddl-tables/pnl_ticks'; +import * as athenaPnl from '../lib/athena-ddl-tables/pnl'; import * as athenaSubaccountUsernames from '../lib/athena-ddl-tables/subaccount_usernames'; import * as athenaSubaccounts from '../lib/athena-ddl-tables/subaccounts'; import * as athenaTendermintEvents from '../lib/athena-ddl-tables/tendermint_events'; @@ -55,6 +56,7 @@ export const tablesToAddToAthena: { [table: string]: AthenaTableDDLQueries } = { perpetual_markets: athenaPerpetualMarkets, perpetual_positions: athenaPerpetualPositions, pnl_ticks: athenaPnlTicks, + pnl: athenaPnl, subaccounts: athenaSubaccounts, tendermint_events: athenaTendermintEvents, trading_rewards: athenaTradingRewards, From 5496cbe63b29875afd4f97834ee2e8d2ef07ed13 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 28 Oct 2025 17:20:28 -0400 Subject: [PATCH 4/5] Push to staging for testing --- .github/workflows/indexer-build-and-push-dev-staging.yml | 1 + indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts | 2 +- .../roundtable/src/tasks/update-research-environment.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/indexer-build-and-push-dev-staging.yml b/.github/workflows/indexer-build-and-push-dev-staging.yml index 87b597c0a62..1bf140983cb 100644 --- a/.github/workflows/indexer-build-and-push-dev-staging.yml +++ b/.github/workflows/indexer-build-and-push-dev-staging.yml @@ -6,6 +6,7 @@ on: # yamllint disable-line rule:truthy - main - 'release/indexer/v[0-9]+.[0-9]+.x' # e.g. release/indexer/v0.1.x - 'release/indexer/v[0-9]+.x' # e.g. release/indexer/v1.x + - new_vault_tables # TODO(DEC-837): Customize github build and push to ECR by service with paths jobs: diff --git a/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts b/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts index e279e61f4c3..d3b931745d2 100644 --- a/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts +++ b/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts @@ -40,4 +40,4 @@ export function generateTable(tablePrefix: string): string { TABLE_NAME, TABLE_COLUMNS, ); -} \ No newline at end of file +} diff --git a/indexer/services/roundtable/src/tasks/update-research-environment.ts b/indexer/services/roundtable/src/tasks/update-research-environment.ts index 929974d5f4d..df3c092bdc2 100644 --- a/indexer/services/roundtable/src/tasks/update-research-environment.ts +++ b/indexer/services/roundtable/src/tasks/update-research-environment.ts @@ -32,8 +32,8 @@ import * as athenaOraclePrices from '../lib/athena-ddl-tables/oracle_prices'; import * as athenaOrders from '../lib/athena-ddl-tables/orders'; import * as athenaPerpetualMarkets from '../lib/athena-ddl-tables/perpetual_markets'; import * as athenaPerpetualPositions from '../lib/athena-ddl-tables/perpetual_positions'; -import * as athenaPnlTicks from '../lib/athena-ddl-tables/pnl_ticks'; import * as athenaPnl from '../lib/athena-ddl-tables/pnl'; +import * as athenaPnlTicks from '../lib/athena-ddl-tables/pnl_ticks'; import * as athenaSubaccountUsernames from '../lib/athena-ddl-tables/subaccount_usernames'; import * as athenaSubaccounts from '../lib/athena-ddl-tables/subaccounts'; import * as athenaTendermintEvents from '../lib/athena-ddl-tables/tendermint_events'; From e2e433cb4cb697ee567bdeefb97fa7c9892e5385 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 30 Oct 2025 14:29:19 -0400 Subject: [PATCH 5/5] Deploy to staging full node --- .github/workflows/protocol-build-and-push-snapshot.yml | 1 + .github/workflows/protocol-build-and-push.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/protocol-build-and-push-snapshot.yml b/.github/workflows/protocol-build-and-push-snapshot.yml index ae94a69a836..2d66e8cdc85 100644 --- a/.github/workflows/protocol-build-and-push-snapshot.yml +++ b/.github/workflows/protocol-build-and-push-snapshot.yml @@ -6,6 +6,7 @@ on: # yamllint disable-line rule:truthy - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x + - new_vault_tables jobs: build-and-push-snapshot-dev: diff --git a/.github/workflows/protocol-build-and-push.yml b/.github/workflows/protocol-build-and-push.yml index 3e9adfcf809..65d25a0b8f9 100644 --- a/.github/workflows/protocol-build-and-push.yml +++ b/.github/workflows/protocol-build-and-push.yml @@ -6,6 +6,7 @@ on: # yamllint disable-line rule:truthy - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x + - new_vault_tables jobs: build-and-push-dev: