Add support for fields tool (#253)

* Add process fields function

* Parse tables from cards

* Combine all metadata processing

* Parallelize metadata processor

* Remove unused metadata process functions

Author: ppsreejith

apps/src/metabase/appState.ts

@@ -1,4 +1,4 @@
-import { addNativeEventListener, RPCs, configs, renderString, getParsedIframeInfo, unsubscribe, captureEvent, GLOBAL_EVENTS, processCards, processDBSchema } from "web";
+import { addNativeEventListener, RPCs, configs, renderString, getParsedIframeInfo, unsubscribe, captureEvent, GLOBAL_EVENTS, processAllMetadata } from "web";
 import { DefaultAppState } from "../base/appState";
 import { MetabaseController } from "./appController";
 import { DB_INFO_DEFAULT, metabaseInternalState } from "./defaultState";
@@ -206,8 +206,7 @@ export class MetabaseState extends DefaultAppState<MetabaseAppState> {
           // Perf caching
           if (!isCancelled()) {
             console.log('Running perf caching')
-            processCards()
-            processDBSchema()
+            processAllMetadata()
             getDatabaseInfo(dbId)
           }
         })

apps/src/metabase/helpers/metabaseAPI.ts

@@ -178,6 +178,10 @@ export const fetchFieldInfo = createAPI<{ field_id: number }>(
   '/api/field/{{field_id}}'
 );
 
+export const fetchDatabaseFields = createAPI<{ db_id: number }>(
+  '/api/database/{{db_id}}/fields'
+);
+
 // Dataset Operations - For running SQL queries
 export const executeDatasetQuery = createAPI<{
   database: number;

apps/src/metabase/helpers/metabaseAPIHelpers.ts

@@ -5,7 +5,7 @@
  * from metabaseAPI.ts and state functions from metabaseStateAPI.ts.
  */
 
-import { map, get, isEmpty, flatMap, filter, sortBy, reverse, pick, omit } from 'lodash';
+import _, { map, get, isEmpty, flatMap, filter, sortBy, reverse, pick, omit } from 'lodash';
 import { getTablesFromSqlRegex, TableAndSchema } from './parseSql';
 import { handlePromise, deterministicSample } from '../../common/utils';
 import { getCurrentUserInfo, getSelectedDbId } from './metabaseStateAPI';
@@ -29,7 +29,8 @@ import {
   fetchTableMetadata,
   fetchModels,
   fetchCard,
-  fetchCards
+  fetchCards,
+  fetchDatabaseFields
 } from './metabaseAPI';
 import { Card, SearchApiResponse } from './types';
 
@@ -262,8 +263,92 @@ export async function getAllCards() {
   });
 
   console.log('Processed cards:', processedCards);
+  const tables: Record<string, TableAndSchema> = {};
+  _.forEach(processedCards, (card) => {
+    getTablesFromSqlRegex(card.dataset_query?.native?.query || '').forEach((table: TableAndSchema) => {
+      const tableKey = `${table.schema}.${table.name}`;
+      if (!tables[tableKey]) {
+        table.count = table.count || 1; // Initialize count if not present
+        tables[tableKey] = table;
+      } else {
+        // If table already exists, merge the counts
+        tables[tableKey].count = (tables[tableKey].count || 0) + (table.count || 0);
+      }
+    })
+  })
+  const relevantTables = _.chain(tables).values().sortBy('count').reverse().value();
+  console.log('Tables from cards:', relevantTables);
+  
+  return { cards: processedCards, tables: relevantTables };
+}
+
+export async function getAllCardsLegacy() {
+  const result = await getAllCards();
+  return result.cards;
+}
+
+export async function getAllFields() {
+  // Get selected database ID
+  const selectedDbId = await getSelectedDbId();
+  
+  if (!selectedDbId) {
+    console.log('[minusx] getAllFields - No database selected');
+    return [];
+  }
+
+  const fields = await handlePromise(
+    fetchDatabaseFields({ db_id: selectedDbId }),
+    "[minusx] Error getting all fields",
+    []
+  );
+  
+  console.log('[minusx] getAllFields - Total fields:', fields.length);
+  
+  // Return top 1000 fields
+  const limitedFields = fields.slice(0, 1000);
+  
+  console.log('[minusx] getAllFields - Returning fields:', limitedFields.length);
+  
+  return limitedFields;
+}
+
+export async function getAllFieldsFiltered(tableNames: string[]) {
+  if (!tableNames || tableNames.length === 0) {
+    console.log('[minusx] getAllFieldsFiltered - No table names provided, returning empty array');
+    return [];
+  }
+
+  // Get selected database ID
+  const selectedDbId = await getSelectedDbId();
+  
+  if (!selectedDbId) {
+    console.log('[minusx] getAllFieldsFiltered - No database selected');
+    return [];
+  }
+
+  const allFields = await handlePromise(
+    fetchDatabaseFields({ db_id: selectedDbId }),
+    "[minusx] Error getting all fields",
+    []
+  );
+  
+  console.log('[minusx] getAllFieldsFiltered - Total fields before filtering:', allFields.length);
+  
+  // Create a set of table names for faster lookup
+  const tableNameSet = new Set(tableNames);
+  
+  // Filter fields to only those belonging to specified tables
+  const filteredFields = filter(allFields, (field) => {
+    const tableName = get(field, 'table_name');
+    const tableSchema = get(field, 'schema');
+    const fullTableName = tableSchema ? `${tableSchema}.${tableName}` : tableName;
+    
+    return tableNameSet.has(tableName) || tableNameSet.has(fullTableName);
+  });
+  
+  console.log('[minusx] getAllFieldsFiltered - Fields after filtering:', filteredFields.length);
 
-  return processedCards;
+  return filteredFields;
 }
 
 export const getAllRelevantModelsForSelectedDb = async (dbId: number, forceRefreshModels: boolean = false): Promise<MetabaseModel[]> => {

apps/src/package.ts

@@ -1,9 +1,10 @@
 export { getAppStateConfigs } from "./appStateConfigs";
 export { applyTableDiffs } from "./common/utils";
 export { getTableContextYAML, filterTablesByCatalog } from "./metabase/helpers/catalog";
-export { getTableData, getDatabaseTablesAndModelsWithoutFields, getAllCards } from "./metabase/helpers/metabaseAPIHelpers";
+export { getTableData, getDatabaseTablesAndModelsWithoutFields, getAllCards, getAllCardsLegacy, getAllFields, getAllFieldsFiltered } from "./metabase/helpers/metabaseAPIHelpers";
 export { fetchModelInfo } from "./metabase/helpers/metabaseAPI";
 export { getAllTemplateTagsInQuery } from "./metabase/helpers/sqlQuery";
 export { getModelsWithFields, getSelectedAndRelevantModels, modifySqlForMetabaseModels, replaceLLMFriendlyIdentifiersInSqlWithModels } from "./metabase/helpers/metabaseModels";
 export { getCurrentQuery, getDashboardState } from "./metabase/helpers/metabaseStateAPI";
-export { subscribeMB, onMBSubscription } from "./metabase/helpers/stateSubscriptions";
+export { subscribeMB, onMBSubscription } from "./metabase/helpers/stateSubscriptions";
+export { getTablesFromSqlRegex } from "./metabase/helpers/parseSql";

web/src/helpers/LLM/remote.ts

@@ -5,7 +5,7 @@ import { getLLMResponse } from '../../app/api'
 import { getApp } from '../app'
 import { getState } from '../../state/store'
 import { unset } from 'lodash'
-import { processCards, processDBSchema } from '../metadataProcessor'
+import { processAllMetadata } from '../metadataProcessor'
 
 export async function planActionsRemote({
   messages,
@@ -29,21 +29,21 @@ export async function planActionsRemote({
     unset(payload, 'tasks')
   }
 
-  const getCardsPromise = processCards()
-  const getDBSchemaPromise = processDBSchema()
+  const getAllMetadataPromise = processAllMetadata()
 
   // Add metadata hashes for analyst mode (when both drMode and analystMode are enabled)
   if (deepResearch !== 'simple') {
     // Check if analyst mode is enabled by getting current state
     const currentState = getState();
     if (currentState.settings.drMode && currentState.settings.analystMode) {
       try {
-        const cardsHash = await getCardsPromise;
-        const dbSchemaHash = await getDBSchemaPromise;
+        const { cardsHash, dbSchemaHash, fieldsHash } = await getAllMetadataPromise;
         // @ts-ignore
         payload.cardsHash = cardsHash;
         // @ts-ignore
         payload.dbSchemaHash = dbSchemaHash;
+        // @ts-ignore
+        payload.fieldsHash = fieldsHash;
         console.log('[minusx] Added metadata hashes to request for analyst mode');
       } catch (error) {
         console.warn('[minusx] Failed to fetch metadata for analyst mode:', error);

web/src/helpers/metadataProcessor.ts

@@ -12,7 +12,9 @@ import { get } from 'lodash';
 import { setMetadataHash } from '../state/settings/reducer';
 import { getState } from '../state/store';
 import { dispatch } from '../state/dispatch';
-import { getAllCards, getDatabaseTablesAndModelsWithoutFields } from '../../../apps/src/metabase/helpers/metabaseAPIHelpers';
+import { getAllCards, getAllCardsLegacy, getDatabaseTablesAndModelsWithoutFields, getAllFields } from '../../../apps/src/metabase/helpers/metabaseAPIHelpers';
+import { fetchDatabaseFields } from '../../../apps/src/metabase/helpers/metabaseAPI';
+import { getSelectedDbId } from '../../../apps/src/metabase/helpers/metabaseStateAPI';
 
 export interface MetadataItem {
   metadata_type: string;
@@ -149,11 +151,93 @@ async function processMetadataWithCaching(
   return currentHash
 }
 
-export async function processCards() {
-  return await processMetadataWithCaching('cards', getAllCards)
-}
-
-export async function processDBSchema() {
-  return await processMetadataWithCaching('dbSchema', getDatabaseTablesAndModelsWithoutFields)
+export async function processAllMetadata() {
+  console.log('[minusx] Starting coordinated metadata processing with parallel API calls...')
+  
+  // Step 1: Start all expensive API calls in parallel
+  console.log('[minusx] Initiating parallel API calls...')
+  const selectedDbId = await getSelectedDbId()
+  
+  if (!selectedDbId) {
+    throw new Error('No database selected for metadata processing')
+  }
+  
+  const [dbSchema, { cards, tables: referencedTables }, allFields] = await Promise.all([
+    getDatabaseTablesAndModelsWithoutFields(),
+    getAllCards(),
+    fetchDatabaseFields({ db_id: selectedDbId })
+  ])
+  
+  console.log('[minusx] All API calls completed. Processing data...')
+  
+  // Step 2: Create sets for efficient lookup of existing tables
+  const existingTableNames = new Set<string>()
+  
+  // Add tables from dbSchema
+  if (dbSchema.tables) {
+    dbSchema.tables.forEach((table: any) => {
+      const tableName = table.name
+      const schemaName = table.schema || dbSchema.default_schema
+      const fullName = schemaName ? `${schemaName}.${tableName}` : tableName
+      
+      existingTableNames.add(tableName)
+      existingTableNames.add(fullName)
+    })
+  }
+  
+  // Add models from dbSchema
+  if (dbSchema.models) {
+    dbSchema.models.forEach((model: any) => {
+      existingTableNames.add(model.name)
+    })
+  }
+  
+  console.log('[minusx] Found existing tables/models:', existingTableNames.size)
+  
+  // Step 3: Find intersection of referenced tables that actually exist
+  const validReferencedTables = referencedTables.filter((table: any) => {
+    const tableName = table.name
+    const schemaName = table.schema
+    const fullName = schemaName ? `${schemaName}.${tableName}` : tableName
+    
+    return existingTableNames.has(tableName) || existingTableNames.has(fullName)
+  })
+  
+  console.log('[minusx] Valid referenced tables:', validReferencedTables.length, 'out of', referencedTables.length)
+  
+  // Step 4: Filter fields in-memory using table names
+  const validTableNames = new Set(validReferencedTables.map((table: any) => {
+    const schemaName = table.schema
+    return schemaName ? `${schemaName}.${table.name}` : table.name
+  }))
+  
+  console.log('[minusx] Filtering fields for', validTableNames.size, 'valid tables...')
+  
+  const filteredFields = allFields.filter((field: any) => {
+    const tableName = get(field, 'table_name')
+    const tableSchema = get(field, 'schema')
+    const fullTableName = tableSchema ? `${tableSchema}.${tableName}` : tableName
+    
+    return validTableNames.has(tableName) || validTableNames.has(fullTableName)
+  })
+  
+  console.log('[minusx] Fields after filtering:', filteredFields.length, 'out of', allFields.length)
+  
+  // Step 5: Process metadata for all three with filtered data
+  console.log('[minusx] Processing metadata with filtered data...')
+  
+  const [cardsHash, dbSchemaHash, fieldsHash] = await Promise.all([
+    processMetadataWithCaching('cards', async () => cards),
+    processMetadataWithCaching('dbSchema', async () => dbSchema),
+    processMetadataWithCaching('fields', async () => filteredFields)
+  ])
+  
+  console.log('[minusx] Coordinated metadata processing complete')
+  
+  return {
+    cardsHash,
+    dbSchemaHash, 
+    fieldsHash
+  }
 }
 

web/src/package.ts

@@ -10,7 +10,7 @@ export { contains } from './helpers/utils';
 export { addCtesToQuery, processSQLWithCtesOrModels } from './helpers/catalogAsModels';
 export { GLOBAL_EVENTS, captureEvent } from './tracking'
 export { getParsedIframeInfo } from './helpers/origin';
-export { processMetadata, processCards, processDBSchema } from './helpers/metadataProcessor';
+export { processMetadata, processAllMetadata } from './helpers/metadataProcessor';
 export { dispatch } from './state/dispatch';
 export { updateIsDevToolsOpen, updateDevToolsTabName, addMemory } from './state/settings/reducer'
 export { setInstructions } from './state/thumbnails/reducer';