Feature/improve metadata processing (#321)

* Add specific function to retrieve db details

* Remove modelFieldsHash

* Remove card model distinction

* Include more fields in card processing

Author: ppsreejith

apps/src/metabase/helpers/analystModeTypes.ts

@@ -45,11 +45,9 @@ const fieldsToRemove = [
   'archived', 
   'collection_position',
   'source_card_id',
-  'result_metadata',
   'creator',
   'initially_published_at',
   'enable_embedding',
-  'collection_id',
   'made_public_by_id',
   'embedding_params',
   'cache_ttl',

apps/src/metabase/helpers/metabaseAPIHelpers.ts

@@ -213,21 +213,18 @@ export async function getAllCardsAndModels(forceRefresh = false, currentDBId: nu
   console.log('[minusx] getAllCards - Total cards:', cards.length);
 
   // Filter cards by database_id only
-  const filteredCardsAndModels = filter(cards, (card) => {
+  const filteredCards = filter(cards, (card) => {
     const databaseId = get(card, 'database_id');
 
     // Filter by selected database only
     const matchesDatabase = selectedDbId && databaseId? databaseId === selectedDbId : true;
 
     return matchesDatabase;
   });
-  console.log('[minusx] getAllCards - Filtered:', filteredCardsAndModels.length);
+  console.log('[minusx] getAllCards - Filtered:', filteredCards.length);
 
-  // split into cards and models
-  const [filteredCards, filteredModels] = partition(filteredCardsAndModels, (card) => get(card, 'type') !== 'model');
 
   console.log('[minusx] Non model cards:', filteredCards.length);
-  console.log('[minusx] Model cards:', filteredModels.length);
   // Calculate relevancy score and sort by it
   const cardsWithRelevancy = filteredCards.map((card) => {
     const viewCount = get(card, 'view_count') || 0;
@@ -261,24 +258,6 @@ export async function getAllCardsAndModels(forceRefresh = false, currentDBId: nu
   const cardsForProcessing = sortedCards.map(card => omit(card, ['relevancy']));
   const processedCards = map(cardsForProcessing, processCard);
 
-  const processedModelFields = filteredModels.flatMap((model) => {
-    const result_metadata = get(model, 'result_metadata', [])
-    if (result_metadata == null) {
-      console.warn("[minusx] model has no result_metadata: ", model)
-      return []
-    }
-    const columns = result_metadata.map((column: ResultMetadataElm) => {
-      return {
-        id: get(column, 'name'),
-        name: get(column, 'name'),
-        type: get(column, 'base_type'),
-        model_id: get(model, 'id'),
-        model_name: get(model, 'name')
-      }
-    })
-    return columns
-  })
-
   console.log('Processed cards:', processedCards);
   const tables: Record<string, TableAndSchema> = {};
   _.forEach(processedCards, (card) => {
@@ -296,7 +275,7 @@ export async function getAllCardsAndModels(forceRefresh = false, currentDBId: nu
   const relevantTables = _.chain(tables).values().sortBy('count').reverse().value();
   console.log('Tables from cards:', relevantTables);
 
-  return { cards: processedCards, tables: relevantTables, modelFields: processedModelFields };
+  return { cards: processedCards, tables: relevantTables };
 }
 
 export async function getAllFields(currentDBId: number) {
@@ -381,18 +360,25 @@ export const getAllRelevantModelsForSelectedDb = async (dbId: number, forceRefre
   return modelsAsTables.filter((model: MetabaseModel) => model.collectionName !== 'mx_internal');
 }
 
-export async function getDatabaseTablesAndModelsWithoutFields(db_id: number, forceRefreshModels: boolean = false, forceRefreshTables: boolean = false): Promise<DatabaseInfoWithTablesAndModels> {
-  // If dbId not provided, get the currently selected database ID
-  const jsonResponse = forceRefreshTables ? await  fetchDatabaseWithTables.refresh({ db_id }) : await fetchDatabaseWithTables({ db_id });
-  const models = await getAllRelevantModelsForSelectedDb(db_id, forceRefreshModels) ;
+export const getRelevantTablesAndDetailsForSelectedDb = async (dbId: number, forceRefreshTables: boolean = false) => {
+  const jsonResponse = forceRefreshTables ? await fetchDatabaseWithTables.refresh({ db_id: dbId }) : await fetchDatabaseWithTables({ db_id: dbId });
   const defaultSchema = getDefaultSchema(jsonResponse);
   const tables = await Promise.all(
-      map(get(jsonResponse, 'tables', []), (table: any) => extractTableInfo(table, false))
+    map(get(jsonResponse, 'tables', []), (table: any) => extractTableInfo(table, false))
   );
 
   return {
-      ...extractDbInfo(jsonResponse, defaultSchema),
-      tables: tables || [],
+    ...extractDbInfo(jsonResponse, defaultSchema),
+    tables: tables || []
+  };
+}
+
+export async function getDatabaseTablesAndModelsWithoutFields(db_id: number, forceRefreshModels: boolean = false, forceRefreshTables: boolean = false): Promise<DatabaseInfoWithTablesAndModels> {
+  const tablesAndDetails = await getRelevantTablesAndDetailsForSelectedDb(db_id, forceRefreshTables);
+  const models = await getAllRelevantModelsForSelectedDb(db_id, forceRefreshModels);
+
+  return {
+      ...tablesAndDetails,
       models: models || []
   };
 }

apps/src/package.ts

@@ -1,7 +1,7 @@
 export { getAppStateConfigs } from "./appStateConfigs";
 export { applyTableDiffs } from "./common/utils";
 export { getTableContextYAML, filterTablesByCatalog } from "./metabase/helpers/catalog";
-export { getAllCardsAndModels, getTableData, getDatabaseTablesAndModelsWithoutFields, getAllCardsAndModels as getAllCards, getAllFields, getAllFieldsFiltered } from "./metabase/helpers/metabaseAPIHelpers";
+export { getAllCardsAndModels, getTableData, getRelevantTablesAndDetailsForSelectedDb, getDatabaseTablesAndModelsWithoutFields, getAllCardsAndModels as getAllCards, getAllFields, getAllFieldsFiltered } from "./metabase/helpers/metabaseAPIHelpers";
 export { fetchModelInfo } from "./metabase/helpers/metabaseAPI";
 export { getAllTemplateTagsInQuery, applySQLEdits, type SQLEdits } from "./metabase/helpers/sqlQuery";
 export { getModelsWithFields, getSelectedAndRelevantModels, modifySqlForMetabaseModels, replaceLLMFriendlyIdentifiersInSqlWithModels } from "./metabase/helpers/metabaseModels";

web/src/helpers/LLM/remote.ts

@@ -42,16 +42,14 @@ export async function planActionsRemote({
     // Check if analyst mode is enabled
     if (currentState.settings.drMode && currentState.settings.analystMode) {
       try {
-        const { cardsHash, dbSchemaHash, fieldsHash, modelFieldsHash, selectedDbId } = await getAllMetadataPromise;
+        const { cardsHash, dbSchemaHash, fieldsHash, selectedDbId } = await getAllMetadataPromise;
         // @ts-ignore
         payload.cardsHash = cardsHash;
         // @ts-ignore
         payload.dbSchemaHash = dbSchemaHash;
         // @ts-ignore
         payload.fieldsHash = fieldsHash;
         // @ts-ignore
-        payload.modelFieldsHash = modelFieldsHash;
-        // @ts-ignore
         payload.selectedDbId = `${selectedDbId}`;
         console.log('[minusx] Added metadata hashes to request for analyst mode');
       } catch (error) {

web/src/helpers/metadataProcessor.ts

@@ -12,7 +12,7 @@ import { get, isEmpty } from 'lodash';
 import { MetadataProcessingResult, MetadataHashInfo, setMetadataHash, setMetadataProcessingCache, clearMetadataProcessingCache } from '../state/settings/reducer';
 import { getState } from '../state/store';
 import { dispatch } from '../state/dispatch';
-import { getAllCardsAndModels, getDatabaseTablesAndModelsWithoutFields, getAllFields } from 'apps'
+import { getAllCardsAndModels, getDatabaseTablesAndModelsWithoutFields, getAllFields, getRelevantTablesAndDetailsForSelectedDb } from 'apps'
 import { fetchDatabaseFields } from '../../../apps/src/metabase/helpers/metabaseAPI';
 import { getSelectedDbId } from '../../../apps/src/metabase/helpers/metabaseStateAPI';
 import { getModelsWithFields } from '../../../apps/src/metabase/helpers/metabaseModels';
@@ -292,8 +292,8 @@ export async function processAllMetadata(forceRefresh:boolean = false, currentDB
   const processingPromise = (async () => {
     try {
 
-      const [dbSchema, { cards, tables: referencedTables, modelFields }, allFields] = await Promise.all([
-        getDatabaseTablesAndModelsWithoutFields(selectedDbId, forceRefresh, forceRefresh),
+      const [dbSchema, { cards }, allFields] = await Promise.all([
+        getRelevantTablesAndDetailsForSelectedDb(selectedDbId, forceRefresh),
         getAllCardsAndModels(forceRefresh, selectedDbId),
         forceRefresh ? fetchDatabaseFields.refresh({ db_id: selectedDbId }) : fetchDatabaseFields({ db_id: selectedDbId })
       ])
@@ -303,15 +303,13 @@ export async function processAllMetadata(forceRefresh:boolean = false, currentDB
 
       console.log('[minusx] Fields after filtering:', filteredFields.length, 'out of', allFields.length)
 
-      console.log("[minusx] modelFields: ", modelFields.length)      
       // Step 5: Process metadata for all four with filtered data
       console.log('[minusx] Processing metadata with filtered data...')
 
-      const [cardsHash, dbSchemaHash, fieldsHash, modelFieldsHash] = await Promise.all([
+      const [cardsHash, dbSchemaHash, fieldsHash] = await Promise.all([
         processMetadataWithCaching('cards', async () => cards, selectedDbId),
         processMetadataWithCaching('dbSchema', async () => dbSchema, selectedDbId),
         processMetadataWithCaching('fields', async () => filteredFields, selectedDbId),
-        processMetadataWithCaching('modelFields', async () => modelFields, selectedDbId)
       ])
 
       console.log('[minusx] Coordinated metadata processing complete')
@@ -320,7 +318,6 @@ export async function processAllMetadata(forceRefresh:boolean = false, currentDB
         cardsHash,
         dbSchemaHash, 
         fieldsHash,
-        modelFieldsHash,
         selectedDbId
       }
 

web/src/state/settings/reducer.ts

@@ -9,7 +9,6 @@ export interface MetadataProcessingResult {
   cardsHash?: string;
   dbSchemaHash?: string;
   fieldsHash?: string;
-  modelFieldsHash?: string;
   selectedDbId?: number;
 }