From 89413c31fcf4fb38ff2e8370b9f1a60b1af9ad5c Mon Sep 17 00:00:00 2001 From: JQQQ Date: Thu, 6 Jun 2024 17:30:08 +1200 Subject: [PATCH 1/7] db size for admin api --- packages/node-core/CHANGELOG.md | 3 +++ .../node-core/src/admin/admin.controller.ts | 10 +++++-- packages/node-core/src/db/sync-helper.ts | 27 +++++++++++++------ .../src/indexer/entities/Metadata.entity.ts | 1 + .../node-core/src/indexer/store.service.ts | 27 +++++++++++++++---- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index fcd9333973..9737c7603e 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Admin api add query `/db_size` to check database size and upsert in metadata + ## [10.4.1] - 2024-06-06 ### Fixed - Fix various issue in monitor service, file naming issue and export admin service from node-core diff --git a/packages/node-core/src/admin/admin.controller.ts b/packages/node-core/src/admin/admin.controller.ts index 4304e93e31..b4c423dcb7 100644 --- a/packages/node-core/src/admin/admin.controller.ts +++ b/packages/node-core/src/admin/admin.controller.ts @@ -15,7 +15,7 @@ import { } from '@nestjs/common'; import {EventEmitter2, OnEvent} from '@nestjs/event-emitter'; import {TargetBlockPayload, RewindPayload, AdminEvent, IndexerEvent} from '../events'; -import {MonitorService, PoiService} from '../indexer'; +import {MonitorService, PoiService, StoreService} from '../indexer'; import {getLogger} from '../logger'; import {timeout} from '../utils'; import {BlockRangeDto, BlockRangeDtoInterface} from './blockRange'; @@ -43,7 +43,8 @@ export class AdminController { constructor( private monitorService: MonitorService, private poiService: PoiService, - private eventEmitter: EventEmitter2 + private eventEmitter: EventEmitter2, + private storeService: StoreService ) {} @Get('index_history/range') @@ -118,6 +119,11 @@ export class AdminController { ); } } + + @Get('db_size') + async getDbSize(): Promise { + return handleServiceCall(() => this.storeService.syncDbSize()); + } } @Injectable() diff --git a/packages/node-core/src/db/sync-helper.ts b/packages/node-core/src/db/sync-helper.ts index ce5d16de8a..7f8b814c1d 100644 --- a/packages/node-core/src/db/sync-helper.ts +++ b/packages/node-core/src/db/sync-helper.ts @@ -159,8 +159,8 @@ export async function getFunctions(sequelize: Sequelize, schema: string, functio information_schema.routines WHERE specific_schema not in ('pg_catalog', 'information_schema') - and routine_type = 'FUNCTION' - and routine_schema = :schema + and routine_type = 'FUNCTION' + and routine_schema = :schema and routine_name = :functionName; `, { @@ -214,10 +214,10 @@ export function createNotifyTrigger(schema: string, table: string): string { return ` DO $$ BEGIN - CREATE TRIGGER "${triggerName}" AFTER INSERT OR UPDATE OR DELETE - ON "${schema}"."${table}" + CREATE TRIGGER "${triggerName}" AFTER INSERT OR UPDATE OR DELETE + ON "${schema}"."${table}" FOR EACH ROW EXECUTE FUNCTION "${schema}".send_notification('${channelName}'); -EXCEPTION +EXCEPTION WHEN duplicate_object THEN RAISE NOTICE 'Trigger already exists. Ignoring...'; END$$; @@ -276,6 +276,17 @@ export function getExistedIndexesQuery(schema: string): string { return `SELECT indexname FROM pg_indexes WHERE schemaname = '${schema}'`; } +export async function getDbSizeQuery(sequelize: Sequelize, schema: string): Promise { + const [size] = (await sequelize.query( + `select sum(pg_total_relation_size(quote_ident(schemaname) || '.' || quote_ident(tablename)))::bigint from pg_tables where schemaname = :schema`, + { + replacements: {schema}, + type: QueryTypes.SELECT, + } + )) as {sum: number}[]; + return Number(size.sum); +} + // SQL improvement const DEFAULT_SQL_EXE_BATCH = 2000; @@ -293,7 +304,7 @@ export const sqlIterator = (tableName: string, sql: string, batch: number = DEFA current_id INT; BEGIN SELECT MIN(id), MAX(id) INTO start_id, end_id FROM ${tableName}; - + IF start_id IS NOT NULL AND end_id IS NOT NULL THEN FOR current_id IN start_id..end_id BY batch_size LOOP ${sql}; @@ -564,9 +575,9 @@ export function generateForeignKeyQuery(attribute: ModelAttributeColumnOptions, DO $$ BEGIN ALTER TABLE "${foreignTable.schema}"."${tableName}" - ADD + ADD CONSTRAINT ${fkey} - FOREIGN KEY (${attribute.field}) + FOREIGN KEY (${attribute.field}) REFERENCES "${foreignTable.schema}"."${foreignTable.tableName}" (${references.key})`; if (attribute.onDelete) { query += ` ON DELETE ${attribute.onDelete}`; diff --git a/packages/node-core/src/indexer/entities/Metadata.entity.ts b/packages/node-core/src/indexer/entities/Metadata.entity.ts index 4c92355eab..90deab09ac 100644 --- a/packages/node-core/src/indexer/entities/Metadata.entity.ts +++ b/packages/node-core/src/indexer/entities/Metadata.entity.ts @@ -29,6 +29,7 @@ export interface MetadataKeys { deployments: string; lastCreatedPoiHeight: number; latestSyncedPoiHeight: number; + dbSize: number; latestPoiWithMmr: string; // Deprecated, keep for poi migration lastPoiHeight: string; // Deprecated, keep for poi migration } diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 2f054c3173..64792d6ed9 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -20,6 +20,7 @@ import { BTREE_GIST_EXTENSION_EXIST_QUERY, createSchemaTrigger, createSchemaTriggerFunction, + getDbSizeQuery, getTriggers, SchemaMigrationService, } from '../db'; @@ -58,7 +59,7 @@ export class StoreService { private _historical?: boolean; private _dbType?: SUPPORT_DB; private _metadataModel?: CacheMetadataModel; - + private _schema?: string; // Should be updated each block private _blockHeight?: number; private _operationStack?: StoreOperations; @@ -103,6 +104,13 @@ export class StoreService { return this._historical; } + async syncDbSize(): Promise { + const dbSize = await getDbSizeQuery(this.sequelize, this.schema); + // It doesn't need to update immediately + this.storeCache.metadata.set('dbSize', dbSize); + return dbSize; + } + private get dbType(): SUPPORT_DB { assert(this._dbType, new NoInitError()); return this._dbType; @@ -113,6 +121,11 @@ export class StoreService { return this._metadataModel; } + private get schema(): string { + assert(this._schema, new NoInitError()); + return this._schema; + } + // Initialize tables and data that isnt' specific to the users data async initCoreTables(schema: string): Promise { if (this.config.proofOfIndex) { @@ -128,6 +141,7 @@ export class StoreService { ); this._dbType = await getDbType(this.sequelize); + this._schema = schema; await this.sequelize.sync(); @@ -283,10 +297,13 @@ export class StoreService { {type: QueryTypes.SELECT} ); - const store = res.reduce(function (total, current) { - total[current.key] = current.value; - return total; - }, {} as {[key: string]: string | boolean}); + const store = res.reduce( + function (total, current) { + total[current.key] = current.value; + return total; + }, + {} as {[key: string]: string | boolean} + ); const useHistorical = store.historicalStateEnabled === undefined ? !disableHistorical : (store.historicalStateEnabled as boolean); From 36aa344c76a76e96b45dacf23d73b6676011bfd6 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 7 Jun 2024 09:25:12 +1200 Subject: [PATCH 2/7] update in query --- packages/query/src/graphql/plugins/GetMetadataPlugin.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts index 4d71463fa9..63403a69c3 100644 --- a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts +++ b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts @@ -33,6 +33,7 @@ const METADATA_TYPES = { deployments: 'string', lastCreatedPoiHeight: 'number', latestSyncedPoiHeight: 'number', + dbSize: 'number', }; const METADATA_KEYS = Object.keys(METADATA_TYPES); @@ -217,6 +218,7 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) unfinalizedBlocks: String lastCreatedPoiHeight: Int latestSyncedPoiHeight: Int + dbSize: Int } type _MetadatasEdge { @@ -236,8 +238,8 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) _metadatas( after: Cursor before: Cursor # distinct: [_mmr_distinct_enum] = null # filter: _MetadataFilter # first: Int - ): # last: Int - # offset: Int + # last: Int + ): # offset: Int # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC] _Metadatas } From a7bd18f5b8595ba7c37afc89e04ee260c266d1a9 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 7 Jun 2024 12:53:10 +1200 Subject: [PATCH 3/7] calculate dbsize and update metadata in same query --- .../node-core/src/admin/admin.controller.ts | 2 +- packages/node-core/src/db/sync-helper.ts | 32 +++++++++++++------ .../src/indexer/entities/Metadata.entity.ts | 2 +- .../node-core/src/indexer/store.service.ts | 23 +++++++++---- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/packages/node-core/src/admin/admin.controller.ts b/packages/node-core/src/admin/admin.controller.ts index b4c423dcb7..ed6a686187 100644 --- a/packages/node-core/src/admin/admin.controller.ts +++ b/packages/node-core/src/admin/admin.controller.ts @@ -121,7 +121,7 @@ export class AdminController { } @Get('db_size') - async getDbSize(): Promise { + async getDbSize(): Promise { return handleServiceCall(() => this.storeService.syncDbSize()); } } diff --git a/packages/node-core/src/db/sync-helper.ts b/packages/node-core/src/db/sync-helper.ts index 7f8b814c1d..dcbe03249a 100644 --- a/packages/node-core/src/db/sync-helper.ts +++ b/packages/node-core/src/db/sync-helper.ts @@ -276,15 +276,29 @@ export function getExistedIndexesQuery(schema: string): string { return `SELECT indexname FROM pg_indexes WHERE schemaname = '${schema}'`; } -export async function getDbSizeQuery(sequelize: Sequelize, schema: string): Promise { - const [size] = (await sequelize.query( - `select sum(pg_total_relation_size(quote_ident(schemaname) || '.' || quote_ident(tablename)))::bigint from pg_tables where schemaname = :schema`, - { - replacements: {schema}, - type: QueryTypes.SELECT, - } - )) as {sum: number}[]; - return Number(size.sum); +export async function getDbSizeAndUpdateMetadata(sequelize: Sequelize, schema: string): Promise { + const [result] = ( + await sequelize.query( + ` + WITH schema_size AS ( + SELECT sum(pg_total_relation_size(quote_ident(schemaname) || '.' || quote_ident(tablename)))::bigint AS size + FROM pg_tables + WHERE schemaname = :schema + ) + UPDATE "${schema}"._metadata + SET value = to_jsonb((SELECT size FROM schema_size)), + "updatedAt" = now() + WHERE key = 'dbSize' + RETURNING (SELECT size FROM schema_size) AS size; + `, + { + replacements: {schema}, + type: QueryTypes.UPDATE, + } + ) + )[0] as unknown as {size: number}[]; + + return BigInt(result.size); } // SQL improvement diff --git a/packages/node-core/src/indexer/entities/Metadata.entity.ts b/packages/node-core/src/indexer/entities/Metadata.entity.ts index 90deab09ac..e02224dee5 100644 --- a/packages/node-core/src/indexer/entities/Metadata.entity.ts +++ b/packages/node-core/src/indexer/entities/Metadata.entity.ts @@ -29,7 +29,7 @@ export interface MetadataKeys { deployments: string; lastCreatedPoiHeight: number; latestSyncedPoiHeight: number; - dbSize: number; + dbSize: bigint; latestPoiWithMmr: string; // Deprecated, keep for poi migration lastPoiHeight: string; // Deprecated, keep for poi migration } diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index 64792d6ed9..49fb5014c5 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -20,7 +20,7 @@ import { BTREE_GIST_EXTENSION_EXIST_QUERY, createSchemaTrigger, createSchemaTriggerFunction, - getDbSizeQuery, + getDbSizeAndUpdateMetadata, getTriggers, SchemaMigrationService, } from '../db'; @@ -36,6 +36,7 @@ import {ISubqueryProject} from './types'; const logger = getLogger('StoreService'); const NULL_MERKEL_ROOT = hexToU8a('0x00'); +const DB_SIZE_CACHE_TIMEOUT = 10 * 60 * 1000; // 10 minutes interface IndexField { entityName: string; @@ -63,6 +64,7 @@ export class StoreService { // Should be updated each block private _blockHeight?: number; private _operationStack?: StoreOperations; + private _lastTimeDbSizeChecked?: number; constructor( private sequelize: Sequelize, @@ -104,11 +106,20 @@ export class StoreService { return this._historical; } - async syncDbSize(): Promise { - const dbSize = await getDbSizeQuery(this.sequelize, this.schema); - // It doesn't need to update immediately - this.storeCache.metadata.set('dbSize', dbSize); - return dbSize; + async syncDbSize(): Promise { + if (!this._lastTimeDbSizeChecked || Date.now() - this._lastTimeDbSizeChecked > DB_SIZE_CACHE_TIMEOUT) { + this._lastTimeDbSizeChecked = Date.now(); + return getDbSizeAndUpdateMetadata(this.sequelize, this.schema); + } else { + return this.storeCache.metadata.find('dbSize').then((cachedDbSize) => { + if (cachedDbSize !== undefined) { + return cachedDbSize; + } else { + this._lastTimeDbSizeChecked = Date.now(); + return getDbSizeAndUpdateMetadata(this.sequelize, this.schema); + } + }); + } } private get dbType(): SUPPORT_DB { From 5dfadaba9663f28c2f8cd3fe09d3d22f53818299 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 7 Jun 2024 12:58:29 +1200 Subject: [PATCH 4/7] tidy up --- packages/query/src/graphql/plugins/GetMetadataPlugin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts index 63403a69c3..af812c8f91 100644 --- a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts +++ b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts @@ -238,8 +238,8 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) _metadatas( after: Cursor before: Cursor # distinct: [_mmr_distinct_enum] = null # filter: _MetadataFilter # first: Int - # last: Int - ): # offset: Int + ): # last: Int + # offset: Int # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC] _Metadatas } From d15df809fb77852e149ed3535d94c959b678a4cd Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 7 Jun 2024 13:02:32 +1200 Subject: [PATCH 5/7] update type --- packages/query/src/graphql/plugins/GetMetadataPlugin.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts index af812c8f91..77e9add539 100644 --- a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts +++ b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts @@ -218,7 +218,7 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) unfinalizedBlocks: String lastCreatedPoiHeight: Int latestSyncedPoiHeight: Int - dbSize: Int + dbSize: BigInt } type _MetadatasEdge { @@ -238,8 +238,8 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) _metadatas( after: Cursor before: Cursor # distinct: [_mmr_distinct_enum] = null # filter: _MetadataFilter # first: Int - ): # last: Int - # offset: Int + # last: Int + ): # offset: Int # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC] _Metadatas } From 7e49cc733da80e5b9628483cc65f81e3882fd360 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 7 Jun 2024 13:27:35 +1200 Subject: [PATCH 6/7] fix --- packages/query/src/graphql/plugins/GetMetadataPlugin.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts index 77e9add539..1703c774ff 100644 --- a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts +++ b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts @@ -33,7 +33,7 @@ const METADATA_TYPES = { deployments: 'string', lastCreatedPoiHeight: 'number', latestSyncedPoiHeight: 'number', - dbSize: 'number', + dbSize: 'string', }; const METADATA_KEYS = Object.keys(METADATA_TYPES); @@ -238,8 +238,8 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) _metadatas( after: Cursor before: Cursor # distinct: [_mmr_distinct_enum] = null # filter: _MetadataFilter # first: Int - # last: Int - ): # offset: Int + # offset: Int + ): # last: Int # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC] _Metadatas } From 25c1d2f947113aea30f90e41051748e3b7e80ca7 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 7 Jun 2024 13:40:55 +1200 Subject: [PATCH 7/7] changelog --- packages/query/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/query/CHANGELOG.md b/packages/query/CHANGELOG.md index fa22619237..984e94c511 100644 --- a/packages/query/CHANGELOG.md +++ b/packages/query/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Add query `dbSize` under metadata (#2430) + ## [2.11.1] - 2024-05-09 ### Fixed - Certain historical queries not appliying block height (#2398)