add modelFields entry in db_metadata for model fields (#275)

* add modelFields entry in db_metadata for model fields

* use the /cards endpoint itself to get model fields

* revert metabaseModels.ts

* handle null result_metadata

Author: kektobiologist

apps/src/metabase/helpers/metabaseAPIHelpers.ts

@@ -5,7 +5,7 @@
  * from metabaseAPI.ts and state functions from metabaseStateAPI.ts.
  */
 
-import _, { map, get, isEmpty, flatMap, filter, sortBy, reverse, pick, omit } from 'lodash';
+import _, { map, get, isEmpty, flatMap, filter, sortBy, reverse, pick, omit, partition } from 'lodash';
 import { getTablesFromSqlRegex, TableAndSchema } from './parseSql';
 import { handlePromise, deterministicSample } from '../../common/utils';
 import { getCurrentUserInfo, getSelectedDbId } from './metabaseStateAPI';
@@ -189,7 +189,16 @@ export async function getDatabases() {
   return await fetchDatabases({}) as DatabaseResponse;
 }
 
-export async function getAllCards(forceRefresh = false) {
+type ResultMetadataElm = {
+  display_name: string;
+  source: string;
+  name: string;
+  base_type: string;
+  semantic_type: string;
+  field_ref: [string, string, { "base-type": string }];
+};
+
+export async function getAllCardsAndModels(forceRefresh = false) {
   const cards = await handlePromise(
     forceRefresh ? fetchCards.refresh({}) : fetchCards({}),
     "[minusx] Error getting all cards",
@@ -207,18 +216,18 @@ export async function getAllCards(forceRefresh = false) {
   console.log('[minusx] getAllCards - Total cards:', cards);
 
   // Filter cards by database_id and last_used_at (last 3 months only)
-  const filteredCards = filter(cards, (card) => {
+  const filteredCardsAndModels = filter(cards, (card) => {
     const lastUsedAt = get(card, 'last_used_at');
     const databaseId = get(card, 'database_id');
 
     // Filter by selected database and last 3 months
     const matchesDatabase = selectedDbId ? databaseId === selectedDbId : true;
     const isRecent = lastUsedAt && lastUsedAt > cutoffDate;
-    const isModel = get(card, 'type') === 'model';
 
-    return matchesDatabase && isRecent && !isModel;
+    return matchesDatabase && isRecent;
   });
-  
+  // split into cards and models
+  const [filteredCards, filteredModels] = partition(filteredCardsAndModels, (card) => get(card, 'type') !== 'model');
   // Sort by view_count descending
   // if length > 1000, limit to 1000
   const sortedCards = reverse(sortBy(filteredCards, 'view_count'));
@@ -261,7 +270,25 @@ export async function getAllCards(forceRefresh = false) {
 
     return cleanCard;
   });
-  
+
+  const processedModelFields = filteredModels.flatMap((model) => {
+    const result_metadata = get(model, 'result_metadata', [])
+    if (result_metadata == null) {
+      console.warn("[minusx] model has no result_metadata: ", model)
+      return []
+    }
+    const columns = result_metadata.map((column: ResultMetadataElm) => {
+      return {
+        id: get(column, 'name'),
+        name: get(column, 'name'),
+        type: get(column, 'base_type'),
+        model_id: get(model, 'id'),
+        model_name: get(model, 'name')
+      }
+    })
+    return columns
+  })
+
   console.log('Processed cards:', processedCards);
   const tables: Record<string, TableAndSchema> = {};
   _.forEach(processedCards, (card) => {
@@ -279,11 +306,11 @@ export async function getAllCards(forceRefresh = false) {
   const relevantTables = _.chain(tables).values().sortBy('count').reverse().value();
   console.log('Tables from cards:', relevantTables);
 
-  return { cards: processedCards, tables: relevantTables };
+  return { cards: processedCards, tables: relevantTables, modelFields: processedModelFields };
 }
 
 export async function getAllCardsLegacy() {
-  const result = await getAllCards();
+  const result = await getAllCardsAndModels();
   return result.cards;
 }
 

apps/src/package.ts

@@ -1,7 +1,7 @@
 export { getAppStateConfigs } from "./appStateConfigs";
 export { applyTableDiffs } from "./common/utils";
 export { getTableContextYAML, filterTablesByCatalog } from "./metabase/helpers/catalog";
-export { getTableData, getDatabaseTablesAndModelsWithoutFields, getAllCards, getAllCardsLegacy, getAllFields, getAllFieldsFiltered } from "./metabase/helpers/metabaseAPIHelpers";
+export { getTableData, getDatabaseTablesAndModelsWithoutFields, getAllCardsAndModels as getAllCards, getAllCardsLegacy, getAllFields, getAllFieldsFiltered } from "./metabase/helpers/metabaseAPIHelpers";
 export { fetchModelInfo } from "./metabase/helpers/metabaseAPI";
 export { getAllTemplateTagsInQuery } from "./metabase/helpers/sqlQuery";
 export { getModelsWithFields, getSelectedAndRelevantModels, modifySqlForMetabaseModels, replaceLLMFriendlyIdentifiersInSqlWithModels } from "./metabase/helpers/metabaseModels";

web/src/helpers/LLM/remote.ts

@@ -37,13 +37,15 @@ export async function planActionsRemote({
     const currentState = getState();
     if (currentState.settings.drMode && currentState.settings.analystMode) {
       try {
-        const { cardsHash, dbSchemaHash, fieldsHash } = await getAllMetadataPromise;
+        const { cardsHash, dbSchemaHash, fieldsHash, modelFieldsHash } = await getAllMetadataPromise;
         // @ts-ignore
         payload.cardsHash = cardsHash;
         // @ts-ignore
         payload.dbSchemaHash = dbSchemaHash;
         // @ts-ignore
         payload.fieldsHash = fieldsHash;
+        // @ts-ignore
+        payload.modelFieldsHash = modelFieldsHash;
         console.log('[minusx] Added metadata hashes to request for analyst mode');
       } catch (error) {
         console.warn('[minusx] Failed to fetch metadata for analyst mode:', error);

web/src/helpers/metadataProcessor.ts

@@ -12,9 +12,10 @@ import { get, isEmpty } from 'lodash';
 import { MetadataProcessingResult, setMetadataHash, setMetadataProcessingCache, clearMetadataProcessingCache } from '../state/settings/reducer';
 import { getState } from '../state/store';
 import { dispatch } from '../state/dispatch';
-import { getAllCards, getAllCardsLegacy, getDatabaseTablesAndModelsWithoutFields, getAllFields } from '../../../apps/src/metabase/helpers/metabaseAPIHelpers';
+import { getAllCardsAndModels, getAllCardsLegacy, getDatabaseTablesAndModelsWithoutFields, getAllFields } from '../../../apps/src/metabase/helpers/metabaseAPIHelpers';
 import { fetchDatabaseFields } from '../../../apps/src/metabase/helpers/metabaseAPI';
 import { getSelectedDbId } from '../../../apps/src/metabase/helpers/metabaseStateAPI';
+import { getModelsWithFields } from '../../../apps/src/metabase/helpers/metabaseModels';
 
 export interface MetadataItem {
   metadata_type: string;
@@ -200,12 +201,11 @@ export async function processAllMetadata(forceRefresh = false) : Promise<Metadat
   const processingPromise = (async () => {
     try {
 
-      const [dbSchema, { cards, tables: referencedTables }, allFields] = await Promise.all([
+      const [dbSchema, { cards, tables: referencedTables, modelFields }, allFields] = await Promise.all([
         getDatabaseTablesAndModelsWithoutFields(selectedDbId, forceRefresh, forceRefresh),
-        getAllCards(forceRefresh),
+        getAllCardsAndModels(forceRefresh),
         forceRefresh ? fetchDatabaseFields.refresh({ db_id: selectedDbId }) : fetchDatabaseFields({ db_id: selectedDbId })
       ])
-      
       console.log('[minusx] All API calls completed. Processing data...')
 
       // Step 2: Create sets for efficient lookup of existing tables
@@ -261,21 +261,24 @@ export async function processAllMetadata(forceRefresh = false) : Promise<Metadat
 
       console.log('[minusx] Fields after filtering:', filteredFields.length, 'out of', allFields.length)
 
-      // Step 5: Process metadata for all three with filtered data
+      console.log("[minusx] modelFields: ", modelFields.length)      
+      // Step 5: Process metadata for all four with filtered data
       console.log('[minusx] Processing metadata with filtered data...')
 
-      const [cardsHash, dbSchemaHash, fieldsHash] = await Promise.all([
+      const [cardsHash, dbSchemaHash, fieldsHash, modelFieldsHash] = await Promise.all([
         processMetadataWithCaching('cards', async () => cards),
         processMetadataWithCaching('dbSchema', async () => dbSchema),
-        processMetadataWithCaching('fields', async () => filteredFields)
+        processMetadataWithCaching('fields', async () => filteredFields),
+        processMetadataWithCaching('modelFields', async () => modelFields)
       ])
 
       console.log('[minusx] Coordinated metadata processing complete')
 
       const result = {
         cardsHash,
         dbSchemaHash, 
-        fieldsHash
+        fieldsHash,
+        modelFieldsHash
       }
 
       // Cache the result for this database ID

web/src/state/settings/reducer.ts

@@ -8,6 +8,7 @@ export interface MetadataProcessingResult {
   cardsHash?: string;
   dbSchemaHash?: string;
   fieldsHash?: string;
+  modelFieldsHash?: string;
 }
 
 interface MetadataProcessingCacheEntry {