diff --git a/.github/workflows/indexer-build-and-push-dev-staging.yml b/.github/workflows/indexer-build-and-push-dev-staging.yml index 87b597c0a62..1bf140983cb 100644 --- a/.github/workflows/indexer-build-and-push-dev-staging.yml +++ b/.github/workflows/indexer-build-and-push-dev-staging.yml @@ -6,6 +6,7 @@ on: # yamllint disable-line rule:truthy - main - 'release/indexer/v[0-9]+.[0-9]+.x' # e.g. release/indexer/v0.1.x - 'release/indexer/v[0-9]+.x' # e.g. release/indexer/v1.x + - new_vault_tables # TODO(DEC-837): Customize github build and push to ECR by service with paths jobs: diff --git a/.github/workflows/protocol-build-and-push-snapshot.yml b/.github/workflows/protocol-build-and-push-snapshot.yml index ae94a69a836..2d66e8cdc85 100644 --- a/.github/workflows/protocol-build-and-push-snapshot.yml +++ b/.github/workflows/protocol-build-and-push-snapshot.yml @@ -6,6 +6,7 @@ on: # yamllint disable-line rule:truthy - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x + - new_vault_tables jobs: build-and-push-snapshot-dev: diff --git a/.github/workflows/protocol-build-and-push.yml b/.github/workflows/protocol-build-and-push.yml index 3e9adfcf809..65d25a0b8f9 100644 --- a/.github/workflows/protocol-build-and-push.yml +++ b/.github/workflows/protocol-build-and-push.yml @@ -6,6 +6,7 @@ on: # yamllint disable-line rule:truthy - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x + - new_vault_tables jobs: build-and-push-dev: diff --git a/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts b/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts new file mode 100644 index 00000000000..7f1de7eab99 --- /dev/null +++ b/indexer/packages/postgres/__tests__/stores/vault-pnl-view.test.ts @@ -0,0 +1,204 @@ +import { DateTime } from 'luxon'; +import { PnlInterval, PnlFromDatabase } from '../../src/types'; +import * as VaultPnlView from '../../src/stores/vault-pnl-view'; +import * as PnlTable from '../../src/stores/pnl-table'; +import * as VaultTable from '../../src/stores/vault-table'; +import * as WalletTable from '../../src/stores/wallet-table'; +import * as SubaccountTable from '../../src/stores/subaccount-table'; +import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; +import { seedData } from '../helpers/mock-generators'; +import { + defaultSubaccountId, + defaultSubaccountIdWithAlternateAddress, + defaultSubaccountWithAlternateAddress, + defaultWallet2, + defaultVault, + defaultSubaccount, +} from '../helpers/constants'; + +describe('VaultPnl store', () => { + beforeAll(async () => { + await migrate(); + }); + + beforeEach(async () => { + await seedData(); + await WalletTable.create(defaultWallet2); + await SubaccountTable.create(defaultSubaccountWithAlternateAddress); + await Promise.all([ + VaultTable.create({ + ...defaultVault, + address: defaultSubaccount.address, + }), + VaultTable.create({ + ...defaultVault, + address: defaultSubaccountWithAlternateAddress.address, + }), + ]); + }); + + afterEach(async () => { + await clearData(); + }); + + afterAll(async () => { + await teardown(); + }); + + it.each([ + { + description: 'Get hourly vault pnl', + interval: PnlInterval.hour, + }, + { + description: 'Get daily vault pnl', + interval: PnlInterval.day, + }, + ])('$description', async ({ + interval, + }: { + interval: PnlInterval, + }) => { + const createdPnl: PnlFromDatabase[] = await setupIntervalPnl(); + + await VaultPnlView.refreshDailyView(); + await VaultPnlView.refreshHourlyView(); + + const pnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + interval, + 7 * 24 * 60 * 60, // 1 week + DateTime.fromISO(createdPnl[8].createdAt).plus({ seconds: 1 }), + ); + + // See setup function for created pnl records. + // Should exclude records that are within the same hour except the first. + const expectedHourlyPnl: PnlFromDatabase[] = [ + createdPnl[7], + createdPnl[5], + createdPnl[2], + createdPnl[0], + ]; + + // Should exclude records that are within the same day except for the first. + const expectedDailyPnl: PnlFromDatabase[] = [ + createdPnl[7], + createdPnl[2], + ]; + + if (interval === PnlInterval.day) { + expect(pnlData).toEqual(expectedDailyPnl); + } else if (interval === PnlInterval.hour) { + expect(pnlData).toEqual(expectedHourlyPnl); + } + }); + + it('Get latest vault pnl', async () => { + await setupIntervalPnl(); + + await VaultPnlView.refreshHourlyView(); + + const latestPnl: PnlFromDatabase[] = await VaultPnlView.getLatestVaultPnl(); + + // Should return the most recent PNL for each subaccount + expect(latestPnl).toHaveLength(2); + + const subaccount1Pnl = latestPnl.find( + (pnl) => pnl.subaccountId === defaultSubaccountId, + ); + const subaccount2Pnl = latestPnl.find( + (pnl) => pnl.subaccountId === defaultSubaccountIdWithAlternateAddress, + ); + + expect(subaccount1Pnl).toBeDefined(); + expect(subaccount1Pnl?.equity).toBe('1100'); + + expect(subaccount2Pnl).toBeDefined(); + expect(subaccount2Pnl?.equity).toBe('200'); + }); + + async function setupIntervalPnl(): Promise { + const currentTime: DateTime = DateTime.utc().startOf('day'); + const tenMinAgo: string = currentTime.minus({ minute: 10 }).toISO(); + const almostTenMinAgo: string = currentTime.minus({ second: 603 }).toISO(); + const twoHoursAgo: string = currentTime.minus({ hour: 2 }).toISO(); + const twoDaysAgo: string = currentTime.minus({ day: 2 }).toISO(); + const monthAgo: string = currentTime.minus({ day: 30 }).toISO(); + + const createdPnl: PnlFromDatabase[] = await PnlTable.createMany([ + { + subaccountId: defaultSubaccountId, + equity: '1100', + createdAt: almostTenMinAgo, + createdAtHeight: '10', + totalPnl: '1200', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1090', + createdAt: tenMinAgo, + createdAtHeight: '8', + totalPnl: '1190', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1080', + createdAt: twoHoursAgo, + createdAtHeight: '6', + totalPnl: '1180', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1070', + createdAt: twoDaysAgo, + createdAtHeight: '4', + totalPnl: '1170', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountId, + equity: '1200', + createdAt: monthAgo, + createdAtHeight: '3', + totalPnl: '1170', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '200', + createdAt: almostTenMinAgo, + createdAtHeight: '10', + totalPnl: '300', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '210', + createdAt: tenMinAgo, + createdAtHeight: '8', + totalPnl: '310', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '220', + createdAt: twoHoursAgo, + createdAtHeight: '6', + totalPnl: '320', + netTransfers: '50', + }, + { + subaccountId: defaultSubaccountIdWithAlternateAddress, + equity: '230', + createdAt: twoDaysAgo, + createdAtHeight: '4', + totalPnl: '330', + netTransfers: '50', + }, + ]); + + return createdPnl; + } +}); diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts new file mode 100644 index 00000000000..6b14bbee20a --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172540_create_vaults_hourly_pnl_v2.ts @@ -0,0 +1,47 @@ +import * as Knex from 'knex'; + +const RAW_VAULTS_PNL_HOURLY_V2_QUERY: string = ` +CREATE MATERIALIZED VIEW IF NOT EXISTS vaults_hourly_pnl_v2 AS +WITH vault_subaccounts AS ( + SELECT subaccounts.id + FROM vaults, subaccounts + WHERE vaults.address = subaccounts.address + AND subaccounts."subaccountNumber" = 0 +), +pnl_subaccounts AS ( + SELECT * FROM vault_subaccounts + UNION + SELECT id FROM subaccounts + WHERE address = 'dydx18tkxrnrkqc2t0lr3zxr5g6a4hdvqksylxqje4r' + AND "subaccountNumber" = 0 +) +SELECT + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" +FROM ( + SELECT + pnl.*, + ROW_NUMBER() OVER ( + partition BY "subaccountId", DATE_TRUNC('hour', "createdAt") + ORDER BY "createdAt" + ) AS r + FROM pnl + WHERE "subaccountId" IN (SELECT * FROM pnl_subaccounts) + AND "createdAt" >= NOW() - interval '604800 second' +) AS pnl_intervals +WHERE r = 1 +ORDER BY "subaccountId"; +`; + +export async function up(knex: Knex): Promise { + await knex.raw(RAW_VAULTS_PNL_HOURLY_V2_QUERY); + await knex.raw('CREATE UNIQUE INDEX ON vaults_hourly_pnl_v2 ("subaccountId", "createdAt");'); +} + +export async function down(knex: Knex): Promise { + await knex.raw('DROP MATERIALIZED VIEW IF EXISTS vaults_hourly_pnl_v2;'); +} diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts new file mode 100644 index 00000000000..4c0a64279c9 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20251027172547_create_vaults_daily_pnl_v2.ts @@ -0,0 +1,47 @@ +import * as Knex from 'knex'; + +const RAW_VAULTS_PNL_DAILY_V2_QUERY: string = ` +CREATE MATERIALIZED VIEW IF NOT EXISTS vaults_daily_pnl_v2 AS +WITH vault_subaccounts AS ( + SELECT subaccounts.id + FROM vaults, subaccounts + WHERE vaults.address = subaccounts.address + AND subaccounts."subaccountNumber" = 0 +), +pnl_subaccounts AS ( + SELECT * FROM vault_subaccounts + UNION + SELECT id FROM subaccounts + WHERE address = 'dydx18tkxrnrkqc2t0lr3zxr5g6a4hdvqksylxqje4r' + AND "subaccountNumber" = 0 +) +SELECT + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" +FROM ( + SELECT + pnl.*, + ROW_NUMBER() OVER ( + partition BY "subaccountId", DATE_TRUNC('day', "createdAt") + ORDER BY "createdAt" + ) AS r + FROM pnl + WHERE "subaccountId" IN (SELECT * FROM pnl_subaccounts) + AND "createdAt" >= NOW() - interval '7776000 second' +) AS pnl_intervals +WHERE r = 1 +ORDER BY "subaccountId"; +`; + +export async function up(knex: Knex): Promise { + await knex.raw(RAW_VAULTS_PNL_DAILY_V2_QUERY); + await knex.raw('CREATE UNIQUE INDEX ON vaults_daily_pnl_v2 ("subaccountId", "createdAt");'); +} + +export async function down(knex: Knex): Promise { + await knex.raw('DROP MATERIALIZED VIEW IF EXISTS vaults_daily_pnl_v2;'); +} diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts index 1efadbbfe9d..f2f980e536e 100644 --- a/indexer/packages/postgres/src/index.ts +++ b/indexer/packages/postgres/src/index.ts @@ -56,6 +56,7 @@ export * as VaultTable from './stores/vault-table'; export * as FundingPaymentsTable from './stores/funding-payments-table'; export * as TurnkeyUsersTable from './stores/turnkey-users-table'; export * as VaultPnlTicksView from './stores/vault-pnl-ticks-view'; +export * as VaultPnlView from './stores/vault-pnl-view'; export * as PermissionApprovalTable from './stores/permission-approval-table'; export * as PnlTable from './stores/pnl-table'; export * as BridgeInformationTable from './stores/bridge-information-table'; diff --git a/indexer/packages/postgres/src/stores/pnl-table.ts b/indexer/packages/postgres/src/stores/pnl-table.ts index d8a211fa52c..0a3d974370e 100644 --- a/indexer/packages/postgres/src/stores/pnl-table.ts +++ b/indexer/packages/postgres/src/stores/pnl-table.ts @@ -373,6 +373,17 @@ export async function create( ).insert(pnlToCreate).returning('*'); } +export async function createMany( + pnls: PnlCreateObject[], + options: Options = { txId: undefined }, +): Promise { + if (!Array.isArray(pnls) || pnls.length === 0) { + return []; + } + const qb = PnlModel.query(Transaction.get(options.txId)); + return qb.insert(pnls).returning('*'); +} + export async function findById( subaccountId: string, createdAt: string, diff --git a/indexer/packages/postgres/src/stores/vault-pnl-view.ts b/indexer/packages/postgres/src/stores/vault-pnl-view.ts new file mode 100644 index 00000000000..5bf24725f7c --- /dev/null +++ b/indexer/packages/postgres/src/stores/vault-pnl-view.ts @@ -0,0 +1,107 @@ +import { DateTime } from 'luxon'; + +import { knexReadReplica } from '../helpers/knex'; +import { rawQuery } from '../helpers/stores-helpers'; +import { PnlFromDatabase, PnlInterval } from '../types'; + +const VAULT_HOURLY_PNL_VIEW: string = 'vaults_hourly_pnl_v2'; +const VAULT_DAILY_PNL_VIEW: string = 'vaults_daily_pnl_v2'; + +/** + * Refresh the hourly vault PNL materialized view. + */ +export async function refreshHourlyView(): Promise { + await rawQuery( + `REFRESH MATERIALIZED VIEW CONCURRENTLY ${VAULT_HOURLY_PNL_VIEW}`, + { + readReplica: false, + }, + ); +} + +/** + * Refresh the daily vault PNL materialized view. + */ +export async function refreshDailyView(): Promise { + await rawQuery( + `REFRESH MATERIALIZED VIEW CONCURRENTLY ${VAULT_DAILY_PNL_VIEW}`, + { + readReplica: false, + }, + ); +} + +/** + * Get vault PNL data for a given interval and time window. + * + * @param interval - The PNL tick interval (hour or day) + * @param timeWindowSeconds - The time window in seconds + * @param earliestDate - The earliest date to fetch data from + * @returns Array of vault PNL records + */ +export async function getVaultsPnl( + interval: PnlInterval, + timeWindowSeconds: number, + earliestDate: DateTime, +): Promise { + const VIEW_BY_INTERVAL: Record = { + [PnlInterval.hour]: VAULT_HOURLY_PNL_VIEW, + [PnlInterval.day]: VAULT_DAILY_PNL_VIEW, + }; + const viewName = VIEW_BY_INTERVAL[interval]; + if (!Number.isFinite(timeWindowSeconds) || timeWindowSeconds <= 0) { + throw new Error('timeWindowSeconds must be a positive number'); + } + const earliest = earliestDate.toUTC().toJSDate(); // lets pg type it as timestamptz + + const result: { + rows: PnlFromDatabase[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" + FROM ${viewName} + WHERE + "createdAt" >= ? AND + "createdAt" > NOW() - make_interval(secs => ?) + ORDER BY "subaccountId", "createdAt"; + `, + [earliest, Math.trunc(timeWindowSeconds)], + ) as unknown as { + rows: PnlFromDatabase[], + }; + + return result.rows; +} + +/** + * Get the latest vault PNL snapshot for each vault. + * + * @returns Array of latest vault PNL records, one per vault + */ +export async function getLatestVaultPnl(): Promise { + const result: { + rows: PnlFromDatabase[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT DISTINCT ON ("subaccountId") + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "createdAtHeight" + FROM ${VAULT_HOURLY_PNL_VIEW} + ORDER BY "subaccountId", "createdAt" DESC; + `, + ) as unknown as { + rows: PnlFromDatabase[], + }; + + return result.rows; +} diff --git a/indexer/packages/postgres/src/types/pnl-types.ts b/indexer/packages/postgres/src/types/pnl-types.ts index eade4df2f17..8cf789ff752 100644 --- a/indexer/packages/postgres/src/types/pnl-types.ts +++ b/indexer/packages/postgres/src/types/pnl-types.ts @@ -15,3 +15,8 @@ export interface PnlCreateObject { netTransfers: string, totalPnl: string, } + +export enum PnlInterval { + hour = 'hour', + day = 'day', +} diff --git a/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts b/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts index 0078dbb74ae..aa72e2e66ca 100644 --- a/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/refresh-vault-pnl.test.ts @@ -1,12 +1,16 @@ import config from '../../src/config'; -import refreshVaulPnlTask from '../../src/tasks/refresh-vault-pnl'; +import refreshVaultPnlTask from '../../src/tasks/refresh-vault-pnl'; import { Settings, DateTime } from 'luxon'; import { BlockTable, + PnlFromDatabase, + PnlInterval, + PnlTable, PnlTickInterval, PnlTicksFromDatabase, PnlTicksTable, VaultPnlTicksView, + VaultPnlView, VaultTable, dbHelpers, testConstants, @@ -40,91 +44,166 @@ describe('refresh-vault-pnl', () => { afterEach(async () => { await dbHelpers.clearData(); - await VaultPnlTicksView.refreshDailyView(); - await VaultPnlTicksView.refreshHourlyView(); + // Refresh both old and new views + await Promise.all([ + VaultPnlTicksView.refreshDailyView(), + VaultPnlTicksView.refreshHourlyView(), + VaultPnlView.refreshDailyView(), + VaultPnlView.refreshHourlyView(), + ]); jest.clearAllMocks(); Settings.now = () => new Date().valueOf(); }); - it('refreshes hourly view if within time window of an hour', async () => { + it('refreshes hourly views (old and new) if within time window of an hour', async () => { Settings.now = () => currentTime.startOf('hour').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS - 1 }, ).valueOf(); - const pnlTick: PnlTicksFromDatabase = await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Verify old view was refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.hour, 86400, currentTime.minus({ day: 1 }), ); - expect(pnlTicks).toEqual([pnlTick]); + + // Verify new view was refreshed + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.hour, + 86400, + currentTime.minus({ day: 1 }), + ); + + // Both views should have the same data + expect(oldPnlTicks).toHaveLength(1); + expect(newPnlData).toHaveLength(1); + + // Verify data matches + expect(oldPnlTicks[0].subaccountId).toEqual(newPnlData[0].subaccountId); + expect(oldPnlTicks[0].equity).toEqual(newPnlData[0].equity); + expect(oldPnlTicks[0].totalPnl).toEqual(newPnlData[0].totalPnl); + expect(oldPnlTicks[0].netTransfers).toEqual(newPnlData[0].netTransfers); }); - it('refreshes daily view if within time window of a day', async () => { + it('refreshes daily views (old and new) if within time window of a day', async () => { Settings.now = () => currentTime.startOf('day').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS - 1 }, ).valueOf(); - const pnlTick: PnlTicksFromDatabase = await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Verify old view was refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.day, 608400, currentTime.minus({ day: 7 }), ); - expect(pnlTicks).toEqual([pnlTick]); + + // Verify new view was refreshed + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.day, + 608400, + currentTime.minus({ day: 7 }), + ); + + // Both views should have the same data + expect(oldPnlTicks).toHaveLength(1); + expect(newPnlData).toHaveLength(1); + + expect(oldPnlTicks[0].subaccountId).toEqual(newPnlData[0].subaccountId); + expect(newPnlData[0].subaccountId).toEqual(testConstants.defaultSubaccountId); }); - it('does not refresh hourly view if outside of time window of an hour', async () => { + it('does not refresh hourly views if outside of time window of an hour', async () => { Settings.now = () => currentTime.startOf('hour').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS + 1 }, ).valueOf(); - await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Neither view should have been refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.hour, 86400, currentTime.minus({ day: 1 }), ); - expect(pnlTicks).toEqual([]); + + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.hour, + 86400, + currentTime.minus({ day: 1 }), + ); + + expect(oldPnlTicks).toEqual([]); + expect(newPnlData).toEqual([]); }); - it('does not refresh daily view if outside time window of a day', async () => { + it('does not refresh daily views if outside time window of a day', async () => { Settings.now = () => currentTime.startOf('day').plus( { milliseconds: config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS + 1 }, ).valueOf(); - await setupPnlTick(); - await refreshVaulPnlTask(); - const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + await setupPnlData(); + + await refreshVaultPnlTask(); + + // Neither view should have been refreshed + const oldPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( PnlTickInterval.day, 608400, currentTime.minus({ day: 7 }), ); - expect(pnlTicks).toEqual([]); + + const newPnlData: PnlFromDatabase[] = await VaultPnlView.getVaultsPnl( + PnlInterval.day, + 608400, + currentTime.minus({ day: 7 }), + ); + + expect(oldPnlTicks).toEqual([]); + expect(newPnlData).toEqual([]); }); - async function setupPnlTick(): Promise { + /** + * Setup PNL data in both old (pnl_ticks) and new (pnl) tables + * to test that both views refresh correctly during migration period. + */ + async function setupPnlData(): Promise { const twoHoursAgo: string = currentTime.minus({ hour: 2 }).toISO(); - await Promise.all([ - BlockTable.create({ - blockHeight: '6', - time: twoHoursAgo, - }), - ]); - const createdTick: PnlTicksFromDatabase = await PnlTicksTable.create( - { - subaccountId: testConstants.defaultSubaccountId, - equity: '1080', - createdAt: twoHoursAgo, - totalPnl: '1180', - netTransfers: '50', - blockHeight: '6', - blockTime: twoHoursAgo, - }, - ); - return createdTick; + + // Create block for old pnl_ticks table (has foreign key to blocks) + await BlockTable.create({ + blockHeight: '6', + time: twoHoursAgo, + }); + + // Create data in old pnl_ticks table + await PnlTicksTable.create({ + subaccountId: testConstants.defaultSubaccountId, + equity: '1080', + createdAt: twoHoursAgo, + totalPnl: '1180', + netTransfers: '50', + blockHeight: '6', + blockTime: twoHoursAgo, + }); + + // Create data in new pnl table + await PnlTable.create({ + subaccountId: testConstants.defaultSubaccountId, + equity: '1080', + createdAt: twoHoursAgo, + createdAtHeight: '6', + totalPnl: '1180', + netTransfers: '50', + }); } }); diff --git a/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts b/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts new file mode 100644 index 00000000000..d3b931745d2 --- /dev/null +++ b/indexer/services/roundtable/src/lib/athena-ddl-tables/pnl.ts @@ -0,0 +1,43 @@ +import { + getAthenaTableCreationStatement, + getExternalAthenaTableCreationStatement, + castToDouble, + castToTimestamp, +} from '../../helpers/sql'; + +const TABLE_NAME: string = 'pnl'; + +const RAW_TABLE_COLUMNS: string = ` + \`subaccountId\` string, + \`equity\` string, + \`totalPnl\` string, + \`netTransfers\` string, + \`createdAt\` string, + \`createdAtHeight\` bigint +`; + +const TABLE_COLUMNS: string = ` + "subaccountId", + ${castToDouble('equity')}, + ${castToDouble('totalPnl')}, + ${castToDouble('netTransfers')}, + ${castToTimestamp('createdAt')}, + "createdAtHeight" +`; + +export function generateRawTable(tablePrefix: string, rdsExportIdentifier: string): string { + return getExternalAthenaTableCreationStatement( + tablePrefix, + rdsExportIdentifier, + TABLE_NAME, + RAW_TABLE_COLUMNS, + ); +} + +export function generateTable(tablePrefix: string): string { + return getAthenaTableCreationStatement( + tablePrefix, + TABLE_NAME, + TABLE_COLUMNS, + ); +} diff --git a/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts b/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts index 635ef5526f5..aba50e4fd92 100644 --- a/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts +++ b/indexer/services/roundtable/src/tasks/refresh-vault-pnl.ts @@ -1,6 +1,7 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; import { VaultPnlTicksView, + VaultPnlView, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; @@ -13,36 +14,78 @@ export default async function runTask(): Promise { const taskStart: number = Date.now(); try { const currentTime: DateTime = DateTime.utc(); + + // Refresh hourly views if (currentTime.diff( currentTime.startOf('hour'), ).toMillis() < config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS) { logger.info({ at: 'refresh-vault-pnl#runTask', - message: 'Refreshing vault hourly pnl view', + message: 'Refreshing vault hourly pnl views (old and new)', currentTime, }); - await VaultPnlTicksView.refreshHourlyView(); + + const hourlyStart: number = Date.now(); + + // Refresh both old and new views in parallel + await Promise.all([ + VaultPnlTicksView.refreshHourlyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed old hourly view (vaults_hourly_pnl)', + }); + }), + VaultPnlView.refreshHourlyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed new hourly view (vaults_hourly_pnl_v2)', + }); + }), + ]); + stats.timing( `${config.SERVICE_NAME}.refresh-vault-pnl.hourly-view.timing`, - Date.now() - taskStart, + Date.now() - hourlyStart, ); } + // Refresh daily views if (currentTime.diff( currentTime.startOf('day'), ).toMillis() < config.TIME_WINDOW_FOR_REFRESH_VAULT_PNL_MS) { - const refreshDailyStart: number = Date.now(); logger.info({ at: 'refresh-vault-pnl#runTask', - message: 'Refreshing vault daily pnl view', + message: 'Refreshing vault daily pnl views (old and new)', currentTime, }); - await VaultPnlTicksView.refreshDailyView(); + + const dailyStart: number = Date.now(); + + // Refresh both old and new views in parallel + await Promise.all([ + VaultPnlTicksView.refreshDailyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed old daily view (vaults_daily_pnl)', + }); + }), + VaultPnlView.refreshDailyView().then(() => { + logger.info({ + at: 'refresh-vault-pnl#runTask', + message: 'Successfully refreshed new daily view (vaults_daily_pnl_v2)', + }); + }), + ]); + stats.timing( `${config.SERVICE_NAME}.refresh-vault-pnl.daily-view.timing`, - Date.now() - refreshDailyStart, + Date.now() - dailyStart, ); } + stats.timing( + `${config.SERVICE_NAME}.refresh-vault-pnl.total.timing`, + Date.now() - taskStart, + ); } catch (error) { logger.error({ at: 'refresh-vault-pnl#runTask', diff --git a/indexer/services/roundtable/src/tasks/update-research-environment.ts b/indexer/services/roundtable/src/tasks/update-research-environment.ts index 6a578dcdec4..df3c092bdc2 100644 --- a/indexer/services/roundtable/src/tasks/update-research-environment.ts +++ b/indexer/services/roundtable/src/tasks/update-research-environment.ts @@ -32,6 +32,7 @@ import * as athenaOraclePrices from '../lib/athena-ddl-tables/oracle_prices'; import * as athenaOrders from '../lib/athena-ddl-tables/orders'; import * as athenaPerpetualMarkets from '../lib/athena-ddl-tables/perpetual_markets'; import * as athenaPerpetualPositions from '../lib/athena-ddl-tables/perpetual_positions'; +import * as athenaPnl from '../lib/athena-ddl-tables/pnl'; import * as athenaPnlTicks from '../lib/athena-ddl-tables/pnl_ticks'; import * as athenaSubaccountUsernames from '../lib/athena-ddl-tables/subaccount_usernames'; import * as athenaSubaccounts from '../lib/athena-ddl-tables/subaccounts'; @@ -55,6 +56,7 @@ export const tablesToAddToAthena: { [table: string]: AthenaTableDDLQueries } = { perpetual_markets: athenaPerpetualMarkets, perpetual_positions: athenaPerpetualPositions, pnl_ticks: athenaPnlTicks, + pnl: athenaPnl, subaccounts: athenaSubaccounts, tendermint_events: athenaTendermintEvents, trading_rewards: athenaTradingRewards,