db size for admin api

packages/node-core/CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Admin api add query `/db_size` to check database size and upsert in metadata 
+
 ## [10.4.1] - 2024-06-06
 ### Fixed
 - Fix various issue in monitor service, file naming issue and export admin service from node-core

packages/node-core/src/admin/admin.controller.ts

@@ -15,7 +15,7 @@ import {
 } from '@nestjs/common';
 import {EventEmitter2, OnEvent} from '@nestjs/event-emitter';
 import {TargetBlockPayload, RewindPayload, AdminEvent, IndexerEvent} from '../events';
-import {MonitorService, PoiService} from '../indexer';
+import {MonitorService, PoiService, StoreService} from '../indexer';
 import {getLogger} from '../logger';
 import {timeout} from '../utils';
 import {BlockRangeDto, BlockRangeDtoInterface} from './blockRange';
@@ -43,7 +43,8 @@ export class AdminController {
   constructor(
     private monitorService: MonitorService,
     private poiService: PoiService,
-    private eventEmitter: EventEmitter2
+    private eventEmitter: EventEmitter2,
+    private storeService: StoreService
   ) {}
 
   @Get('index_history/range')
@@ -118,6 +119,11 @@ export class AdminController {
       );
     }
   }
+
+  @Get('db_size')
+  async getDbSize(): Promise<bigint> {
+    return handleServiceCall(() => this.storeService.syncDbSize());
+  }
 }
 
 @Injectable()

packages/node-core/src/db/sync-helper.ts

@@ -159,8 +159,8 @@ export async function getFunctions(sequelize: Sequelize, schema: string, functio
          information_schema.routines
      WHERE
          specific_schema not in ('pg_catalog', 'information_schema')
-       and routine_type = 'FUNCTION' 
-       and routine_schema = :schema 
+       and routine_type = 'FUNCTION'
+       and routine_schema = :schema
        and routine_name = :functionName;
     `,
     {
@@ -214,10 +214,10 @@ export function createNotifyTrigger(schema: string, table: string): string {
   return `
 DO $$
 BEGIN
-    CREATE TRIGGER "${triggerName}" AFTER INSERT OR UPDATE OR DELETE 
-    ON "${schema}"."${table}" 
+    CREATE TRIGGER "${triggerName}" AFTER INSERT OR UPDATE OR DELETE
+    ON "${schema}"."${table}"
     FOR EACH ROW EXECUTE FUNCTION "${schema}".send_notification('${channelName}');
-EXCEPTION   
+EXCEPTION
     WHEN duplicate_object THEN
         RAISE NOTICE 'Trigger already exists. Ignoring...';
 END$$;
@@ -276,6 +276,31 @@ export function getExistedIndexesQuery(schema: string): string {
   return `SELECT indexname FROM pg_indexes WHERE schemaname = '${schema}'`;
 }
 
+export async function getDbSizeAndUpdateMetadata(sequelize: Sequelize, schema: string): Promise<bigint> {
+  const [result] = (
+    await sequelize.query(
+      `
+      WITH schema_size AS (
+        SELECT sum(pg_total_relation_size(quote_ident(schemaname) || '.' || quote_ident(tablename)))::bigint AS size
+      FROM pg_tables
+      WHERE schemaname = :schema
+        )
+      UPDATE "${schema}"._metadata
+      SET value = to_jsonb((SELECT size FROM schema_size)),
+          "updatedAt" = now()
+      WHERE key = 'dbSize'
+        RETURNING (SELECT size FROM schema_size) AS size;
+    `,
+      {
+        replacements: {schema},
+        type: QueryTypes.UPDATE,
+      }
+    )
+  )[0] as unknown as {size: number}[];
+
+  return BigInt(result.size);
+}
+
 // SQL improvement
 const DEFAULT_SQL_EXE_BATCH = 2000;
 
@@ -293,7 +318,7 @@ export const sqlIterator = (tableName: string, sql: string, batch: number = DEFA
     current_id INT;
   BEGIN
     SELECT MIN(id), MAX(id) INTO start_id, end_id FROM ${tableName};
-    
+
     IF start_id IS NOT NULL AND end_id IS NOT NULL THEN
         FOR current_id IN start_id..end_id BY batch_size LOOP
             ${sql};
@@ -564,9 +589,9 @@ export function generateForeignKeyQuery(attribute: ModelAttributeColumnOptions,
   DO $$
   BEGIN
     ALTER TABLE "${foreignTable.schema}"."${tableName}"
-      ADD 
+      ADD
       CONSTRAINT ${fkey}
-      FOREIGN KEY (${attribute.field}) 
+      FOREIGN KEY (${attribute.field})
       REFERENCES "${foreignTable.schema}"."${foreignTable.tableName}" (${references.key})`;
   if (attribute.onDelete) {
     query += ` ON DELETE ${attribute.onDelete}`;

packages/node-core/src/indexer/entities/Metadata.entity.ts

@@ -29,6 +29,7 @@ export interface MetadataKeys {
   deployments: string;
   lastCreatedPoiHeight: number;
   latestSyncedPoiHeight: number;
+  dbSize: bigint;
   latestPoiWithMmr: string; // Deprecated, keep for poi migration
   lastPoiHeight: string; // Deprecated, keep for poi migration
 }

packages/node-core/src/indexer/store.service.ts

@@ -20,6 +20,7 @@ import {
   BTREE_GIST_EXTENSION_EXIST_QUERY,
   createSchemaTrigger,
   createSchemaTriggerFunction,
+  getDbSizeAndUpdateMetadata,
   getTriggers,
   SchemaMigrationService,
 } from '../db';
@@ -35,6 +36,7 @@ import {ISubqueryProject} from './types';
 
 const logger = getLogger('StoreService');
 const NULL_MERKEL_ROOT = hexToU8a('0x00');
+const DB_SIZE_CACHE_TIMEOUT = 10 * 60 * 1000; // 10 minutes
 
 interface IndexField {
   entityName: string;
@@ -58,10 +60,11 @@ export class StoreService {
   private _historical?: boolean;
   private _dbType?: SUPPORT_DB;
   private _metadataModel?: CacheMetadataModel;
-
+  private _schema?: string;
   // Should be updated each block
   private _blockHeight?: number;
   private _operationStack?: StoreOperations;
+  private _lastTimeDbSizeChecked?: number;
 
   constructor(
     private sequelize: Sequelize,
@@ -103,6 +106,22 @@ export class StoreService {
     return this._historical;
   }
 
+  async syncDbSize(): Promise<bigint> {
+    if (!this._lastTimeDbSizeChecked || Date.now() - this._lastTimeDbSizeChecked > DB_SIZE_CACHE_TIMEOUT) {
+      this._lastTimeDbSizeChecked = Date.now();
+      return getDbSizeAndUpdateMetadata(this.sequelize, this.schema);
+    } else {
+      return this.storeCache.metadata.find('dbSize').then((cachedDbSize) => {
+        if (cachedDbSize !== undefined) {
+          return cachedDbSize;
+        } else {
+          this._lastTimeDbSizeChecked = Date.now();
+          return getDbSizeAndUpdateMetadata(this.sequelize, this.schema);
+        }
+      });
+    }
+  }
+
   private get dbType(): SUPPORT_DB {
     assert(this._dbType, new NoInitError());
     return this._dbType;
@@ -113,6 +132,11 @@ export class StoreService {
     return this._metadataModel;
   }
 
+  private get schema(): string {
+    assert(this._schema, new NoInitError());
+    return this._schema;
+  }
+
   // Initialize tables and data that isnt' specific to the users data
   async initCoreTables(schema: string): Promise<void> {
     if (this.config.proofOfIndex) {
@@ -128,6 +152,7 @@ export class StoreService {
     );
 
     this._dbType = await getDbType(this.sequelize);
+    this._schema = schema;
 
     await this.sequelize.sync();
 
@@ -283,10 +308,13 @@ export class StoreService {
           {type: QueryTypes.SELECT}
         );
 
-        const store = res.reduce(function (total, current) {
-          total[current.key] = current.value;
-          return total;
-        }, {} as {[key: string]: string | boolean});
+        const store = res.reduce(
+          function (total, current) {
+            total[current.key] = current.value;
+            return total;
+          },
+          {} as {[key: string]: string | boolean}
+        );
 
         const useHistorical =
           store.historicalStateEnabled === undefined ? !disableHistorical : (store.historicalStateEnabled as boolean);

packages/query/src/graphql/plugins/GetMetadataPlugin.ts

@@ -33,6 +33,7 @@ const METADATA_TYPES = {
   deployments: 'string',
   lastCreatedPoiHeight: 'number',
   latestSyncedPoiHeight: 'number',
+  dbSize: 'number',
 };
 
 const METADATA_KEYS = Object.keys(METADATA_TYPES);
@@ -217,6 +218,7 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options)
         unfinalizedBlocks: String
         lastCreatedPoiHeight: Int
         latestSyncedPoiHeight: Int
+        dbSize: BigInt
       }
 
       type _MetadatasEdge {
@@ -236,8 +238,8 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options)
         _metadatas(
           after: Cursor
           before: Cursor # distinct: [_mmr_distinct_enum] = null # filter: _MetadataFilter # first: Int
-        ): # last: Int
-        # offset: Int
+          # last: Int
+        ): # offset: Int
         # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC]
         _Metadatas
       }