Reduce event counts (#256)

* Remove empty notification set events

* Remove polling action from notifications

* Only refresh metadata if dbId has changed

* Add types to processAllMetadata fn

* Cache and re-use metadataProcessingResult

* Ensure metadataProcessing isn't triggered >2 times

* Keep metadataProcessor cache checks synchronous

* Remove unused rpc call logging

* Purge old cache entries

* Remove stale cache entries

* Increase notification polling interval to 5 mins

Author: ppsreejith

apps/src/metabase/appState.ts

@@ -204,7 +204,7 @@ export class MetabaseState extends DefaultAppState<MetabaseAppState> {
             }
           }))
           // Perf caching
-          if (!isCancelled()) {
+          if (!isCancelled() && dbId !== oldDbId) {
             console.log('Running perf caching')
             processAllMetadata()
             getDatabaseInfo(dbId)

web/src/app/rpc.ts

@@ -67,7 +67,7 @@ export const setMinusxMode = (mode: string) =>
 export const toggleMinusXRoot = (
   className: ValidMinusxRootClass,
   value?: boolean
-) => sendMessage('toggleMinusXRoot', [className, value], { log_rpc: true })
+) => sendMessage('toggleMinusXRoot', [className, value], { log_rpc: false })
 export const captureVisibleTab = () =>
   sendMessage('captureVisibleTab', [], { log_rpc: true })
 export const getElementScreenCapture = (selector: QuerySelector) =>
@@ -107,11 +107,11 @@ export const fetchData = (
   })
 export const queryURL = () => sendMessage('queryURL', [])
 export const getMetabaseState = (path: Parameters<typeof get>[1]) =>
-  sendMessage('getMetabaseState', [path], { log_rpc: true })
+  sendMessage('getMetabaseState', [path], { log_rpc: false })
 export const dispatchMetabaseAction = (type: string, payload?: any) =>
   sendMessage('dispatchMetabaseAction', [type, payload], { log_rpc: true, timeout: 1000 })
 export const getSelectedTextOnEditor = () =>
-  sendMessage('getSelectedTextOnEditor', [], { log_rpc: true })
+  sendMessage('getSelectedTextOnEditor', [], { log_rpc: false })
 export const subscribeMetabaseState = (path: string) =>
   sendMessage('subscribeMetabaseState', [path], { log_rpc: true }) as unknown as Promise<number>
 export const getJupyterState = (mode?: string) =>
@@ -126,7 +126,7 @@ export const getPosthogAppContext = (path: Parameters<typeof get>[1]) =>
 export const setTextPosthog = (selector: QuerySelector, value: string = '') =>
   sendMessage('setTextPosthog', [selector, value], { log_rpc: true })
 export const attachMutationListener = (domQueryMap: DOMQueryMap) =>
-  sendMessage('attachMutationListener', [domQueryMap], { log_rpc: true })
+  sendMessage('attachMutationListener', [domQueryMap], { log_rpc: false })
 export const detachMutationListener = (id: number) =>
   sendMessage('detachMutationListener', [id], { log_rpc: true })
 export const forwardToTab = (tool: string, message: string) =>
@@ -166,11 +166,11 @@ export const gsheetSetUserToken = (token: string) =>
 
 export const attachEventsListener = (
   selector: QuerySelector, events?: string[]
-) => sendMessage('attachEventsListener', [selector, events], { log_rpc: true })
+) => sendMessage('attachEventsListener', [selector, events], { log_rpc: false })
 
 export const addNativeElements = (
   selector: QuerySelector, htmlElement: HTMLJSONNode, attachType: AttachType='lastChild'
-) => sendMessage('addNativeElements', [selector, htmlElement, attachType], { log_rpc: true })
+) => sendMessage('addNativeElements', [selector, htmlElement, attachType], { log_rpc: false })
 
 export const startRecording = () => sendMessage('startRecording', [])
 export const stopRecording = () => sendMessage('stopRecording', [])

web/src/cache/cache.ts

@@ -94,4 +94,78 @@ export function memoize<T extends AsyncFunction>(
   };
 
   return memoized as MemoizedFn<T>;
-}
+}
+
+// Debug function to list all cache entries and clean up old ones
+async function clearStaleEntries() {
+  const { openDB } = await import('idb');
+  const db = await openDB('minusx-cache-db', 1);
+  
+  // First, get all entries
+  const tx = db.transaction('cache', 'readonly');
+  const store = tx.objectStore('cache');
+  const allEntries = await store.getAll();
+  const allKeys = await store.getAllKeys();
+
+  const FOUR_WEEKS_MS = 28 * 24 * 60 * 60 * 1000; // 4 weeks in milliseconds
+  const now = Date.now();
+  
+  const entries = allEntries.map((entry, index) => {
+    const key = allKeys[index];
+    const hasCreatedAt = entry.createdAt && typeof entry.createdAt === 'number';
+    
+    if (!hasCreatedAt) {
+      return {
+        key,
+        createdAt: 'MISSING',
+        ageInHours: 'N/A',
+        dataSize: JSON.stringify(entry.data).length,
+        isStale: true, // Remove entries without createdAt
+        reason: 'Missing createdAt'
+      };
+    }
+    
+    return {
+      key,
+      createdAt: new Date(entry.createdAt).toISOString(),
+      ageInHours: ((now - entry.createdAt) / (1000 * 60 * 60)).toFixed(2),
+      dataSize: JSON.stringify(entry.data).length,
+      isStale: (now - entry.createdAt) > FOUR_WEEKS_MS,
+      reason: (now - entry.createdAt) > FOUR_WEEKS_MS ? 'Older than 4 weeks' : 'Fresh'
+    };
+  });
+
+  // Find stale entries (older than 4 weeks OR missing createdAt)
+  const staleEntries = entries.filter(entry => entry.isStale);
+  
+  if (staleEntries.length > 0) {
+    console.log(`Found ${staleEntries.length} stale entries (older than 4 weeks or missing createdAt), removing...`);
+    console.log('Breakdown:', staleEntries.reduce((acc, entry) => {
+      acc[entry.reason] = (acc[entry.reason] || 0) + 1;
+      return acc;
+    }, {} as Record<string, number>));
+    
+    // Delete stale entries
+    const deleteTx = db.transaction('cache', 'readwrite');
+    const deleteStore = deleteTx.objectStore('cache');
+    
+    for (const staleEntry of staleEntries) {
+      await deleteStore.delete(staleEntry.key);
+    }
+    
+    await deleteTx.done;
+    console.log(`Removed ${staleEntries.length} stale cache entries`);
+  }
+  
+  // Show remaining entries
+  const remainingEntries = entries.filter(entry => !entry.isStale);
+  console.table(remainingEntries);
+  
+  return {
+    total: entries.length,
+    remaining: remainingEntries.length,
+    removed: staleEntries.length,
+    entries: remainingEntries
+  };
+}
+clearStaleEntries()

web/src/helpers/documentSubscription.ts

@@ -19,9 +19,6 @@ export const unsubscribe = async (id: number) => {
 
 export const onSubscription = (payload: SubscriptionPayload) => {
     const { id, url } = payload
-    captureEvent(GLOBAL_EVENTS.subscription, {
-        subscription_id: id, url
-    })
     if (!(id in listeners)) {
         return
     }

web/src/helpers/metadataProcessor.ts

@@ -9,7 +9,7 @@ import axios from 'axios';
 import { configs } from '../constants';
 import { getOrigin } from './origin';
 import { get } from 'lodash';
-import { setMetadataHash } from '../state/settings/reducer';
+import { MetadataProcessingResult, setMetadataHash, setMetadataProcessingCache, clearMetadataProcessingCache } from '../state/settings/reducer';
 import { getState } from '../state/store';
 import { dispatch } from '../state/dispatch';
 import { getAllCards, getAllCardsLegacy, getDatabaseTablesAndModelsWithoutFields, getAllFields } from '../../../apps/src/metabase/helpers/metabaseAPIHelpers';
@@ -70,6 +70,9 @@ async function calculateMetadataHash(metadataType: string, metadataValue: any, v
 // Global map to track ongoing uploads by hash
 const ongoingUploads = new Map<string, Promise<string>>();
 
+// Global map to track ongoing metadata processing by dbId
+const ongoingMetadataProcessing = new Map<number, Promise<MetadataProcessingResult>>();
+
 /**
  * Generic function to upload any metadata type to the backend
  * @param metadataType The type of metadata (e.g., 'cards', 'dbSchema')
@@ -151,7 +154,7 @@ async function processMetadataWithCaching(
   return currentHash
 }
 
-export async function processAllMetadata() {
+export async function processAllMetadata() : Promise<MetadataProcessingResult> {
   console.log('[minusx] Starting coordinated metadata processing with parallel API calls...')
 
   // Step 1: Start all expensive API calls in parallel
@@ -162,82 +165,126 @@ export async function processAllMetadata() {
     throw new Error('No database selected for metadata processing')
   }
 
-  const [dbSchema, { cards, tables: referencedTables }, allFields] = await Promise.all([
-    getDatabaseTablesAndModelsWithoutFields(),
-    getAllCards(),
-    fetchDatabaseFields({ db_id: selectedDbId })
-  ])
-  
-  console.log('[minusx] All API calls completed. Processing data...')
-  
-  // Step 2: Create sets for efficient lookup of existing tables
-  const existingTableNames = new Set<string>()
+  // Check cache for this database ID first (synchronous)
+  const currentState = getState()
+  const cacheEntry = currentState.settings.metadataProcessingCache[selectedDbId]
 
-  // Add tables from dbSchema
-  if (dbSchema.tables) {
-    dbSchema.tables.forEach((table: any) => {
-      const tableName = table.name
-      const schemaName = table.schema || dbSchema.default_schema
-      const fullName = schemaName ? `${schemaName}.${tableName}` : tableName
-      
-      existingTableNames.add(tableName)
-      existingTableNames.add(fullName)
-    })
+  if (cacheEntry) {
+    const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000
+    const isStale = Date.now() - cacheEntry.timestamp > SEVEN_DAYS_MS
+    
+    if (!isStale) {
+      console.log(`[minusx] Using cached metadata for database ${selectedDbId}`)
+      return cacheEntry.result
+    } else {
+      console.log(`[minusx] Cached metadata for database ${selectedDbId} is stale, clearing cache`)
+      // Clear stale cache entry using proper Redux action
+      dispatch(clearMetadataProcessingCache(selectedDbId))
+    }
   }
 
-  // Add models from dbSchema
-  if (dbSchema.models) {
-    dbSchema.models.forEach((model: any) => {
-      existingTableNames.add(model.name)
-    })
+  // Check if processing is already in progress for this database ID
+  if (ongoingMetadataProcessing.has(selectedDbId)) {
+    console.log(`[minusx] Metadata processing already in progress for database ${selectedDbId}, returning existing promise`)
+    return await ongoingMetadataProcessing.get(selectedDbId)!
   }
 
-  console.log('[minusx] Found existing tables/models:', existingTableNames.size)
-  
-  // Step 3: Find intersection of referenced tables that actually exist
-  const validReferencedTables = referencedTables.filter((table: any) => {
-    const tableName = table.name
-    const schemaName = table.schema
-    const fullName = schemaName ? `${schemaName}.${tableName}` : tableName
-    
-    return existingTableNames.has(tableName) || existingTableNames.has(fullName)
-  })
-  
-  console.log('[minusx] Valid referenced tables:', validReferencedTables.length, 'out of', referencedTables.length)
-  
-  // Step 4: Filter fields in-memory using table names
-  const validTableNames = new Set(validReferencedTables.map((table: any) => {
-    const schemaName = table.schema
-    return schemaName ? `${schemaName}.${table.name}` : table.name
-  }))
-  
-  console.log('[minusx] Filtering fields for', validTableNames.size, 'valid tables...')
-  
-  const filteredFields = allFields.filter((field: any) => {
-    const tableName = get(field, 'table_name')
-    const tableSchema = get(field, 'schema')
-    const fullTableName = tableSchema ? `${tableSchema}.${tableName}` : tableName
-    
-    return validTableNames.has(tableName) || validTableNames.has(fullTableName)
-  })
-  
-  console.log('[minusx] Fields after filtering:', filteredFields.length, 'out of', allFields.length)
-  
-  // Step 5: Process metadata for all three with filtered data
-  console.log('[minusx] Processing metadata with filtered data...')
+  // Create and store the processing promise
+  const processingPromise = (async () => {
+    try {
+      
+      const [dbSchema, { cards, tables: referencedTables }, allFields] = await Promise.all([
+        getDatabaseTablesAndModelsWithoutFields(),
+        getAllCards(),
+        fetchDatabaseFields({ db_id: selectedDbId })
+      ])
+      
+      console.log('[minusx] All API calls completed. Processing data...')
+      
+      // Step 2: Create sets for efficient lookup of existing tables
+      const existingTableNames = new Set<string>()
+      
+      // Add tables from dbSchema
+      if (dbSchema.tables) {
+        dbSchema.tables.forEach((table: any) => {
+          const tableName = table.name
+          const schemaName = table.schema || dbSchema.default_schema
+          const fullName = schemaName ? `${schemaName}.${tableName}` : tableName
+          
+          existingTableNames.add(tableName)
+          existingTableNames.add(fullName)
+        })
+      }
+      
+      // Add models from dbSchema
+      if (dbSchema.models) {
+        dbSchema.models.forEach((model: any) => {
+          existingTableNames.add(model.name)
+        })
+      }
+      
+      console.log('[minusx] Found existing tables/models:', existingTableNames.size)
+      
+      // Step 3: Find intersection of referenced tables that actually exist
+      const validReferencedTables = referencedTables.filter((table: any) => {
+        const tableName = table.name
+        const schemaName = table.schema
+        const fullName = schemaName ? `${schemaName}.${tableName}` : tableName
+        
+        return existingTableNames.has(tableName) || existingTableNames.has(fullName)
+      })
+      
+      console.log('[minusx] Valid referenced tables:', validReferencedTables.length, 'out of', referencedTables.length)
+      
+      // Step 4: Filter fields in-memory using table names
+      const validTableNames = new Set(validReferencedTables.map((table: any) => {
+        const schemaName = table.schema
+        return schemaName ? `${schemaName}.${table.name}` : table.name
+      }))
+      
+      console.log('[minusx] Filtering fields for', validTableNames.size, 'valid tables...')
+      
+      const filteredFields = allFields.filter((field: any) => {
+        const tableName = get(field, 'table_name')
+        const tableSchema = get(field, 'schema')
+        const fullTableName = tableSchema ? `${tableSchema}.${tableName}` : tableName
+        
+        return validTableNames.has(tableName) || validTableNames.has(fullTableName)
+      })
+      
+      console.log('[minusx] Fields after filtering:', filteredFields.length, 'out of', allFields.length)
+      
+      // Step 5: Process metadata for all three with filtered data
+      console.log('[minusx] Processing metadata with filtered data...')
+      
+      const [cardsHash, dbSchemaHash, fieldsHash] = await Promise.all([
+        processMetadataWithCaching('cards', async () => cards),
+        processMetadataWithCaching('dbSchema', async () => dbSchema),
+        processMetadataWithCaching('fields', async () => filteredFields)
+      ])
+      
+      console.log('[minusx] Coordinated metadata processing complete')
+      
+      const result = {
+        cardsHash,
+        dbSchemaHash, 
+        fieldsHash
+      }
 
-  const [cardsHash, dbSchemaHash, fieldsHash] = await Promise.all([
-    processMetadataWithCaching('cards', async () => cards),
-    processMetadataWithCaching('dbSchema', async () => dbSchema),
-    processMetadataWithCaching('fields', async () => filteredFields)
-  ])
+      // Cache the result for this database ID
+      dispatch(setMetadataProcessingCache({ dbId: selectedDbId, result }))
+      console.log(`[minusx] Cached metadata processing result for database ${selectedDbId}`)
+      
+      return result
+    } finally {
+      // Clean up the ongoing processing tracking
+      ongoingMetadataProcessing.delete(selectedDbId)
+    }
+  })()
 
-  console.log('[minusx] Coordinated metadata processing complete')
+  // Store the promise in the map
+  ongoingMetadataProcessing.set(selectedDbId, processingPromise)
 
-  return {
-    cardsHash,
-    dbSchemaHash, 
-    fieldsHash
-  }
+  return await processingPromise
 }