add dashboard models into entities; also fix lastSQL generated in new-tab flow on dashboard page (#191)

Author: kektobiologist

apps/src/metabase/helpers/dashboard/appState.ts

@@ -2,14 +2,16 @@ import { DashboardInfo, DashboardMetabaseState } from './types';
 import _, { forEach, reduce, template, values } from 'lodash';
 import { MetabaseAppStateDashboard,  MetabaseAppStateType} from '../DOMToState';
 import { getTablesWithFields } from '../getDatabaseSchema';
-import { getDatabaseInfo, getFieldResolvedName } from '../metabaseAPIHelpers';
+import { getAllRelevantModelsForSelectedDb, getDatabaseInfo, getFieldResolvedName } from '../metabaseAPIHelpers';
 import { getDashboardState, getSelectedDbId } from '../metabaseStateAPI';
 import { getParsedIframeInfo, RPCs } from 'web';
 import { getSQLFromMBQL } from '../metabaseAPI';
 import { metabaseToMarkdownTable } from '../operations';
 import { find, get } from 'lodash';
 import { getTablesFromSqlRegex, TableAndSchema } from '../parseSql';
 import { getTableContextYAML } from '../catalog';
+import { MetabaseModel } from '../metabaseAPITypes';
+import { getModelsFromSql, getModelsWithFields, modifySqlForMetabaseModels, replaceLLMFriendlyIdentifiersInSqlWithModels } from '../metabaseModels';
 
 // Removed: const { getMetabaseState } = RPCs - using centralized state functions instead
 
@@ -194,7 +196,8 @@ async function substituteParameters(
   sql: string, 
   dashcard: DashboardMetabaseState['dashcards'][0],
   dashboardParamFields: DashboardMetabaseState['dashboards'][0]['param_fields'],
-  parameterValues: DashboardMetabaseState['parameterValues']) {
+  parameterValues: DashboardMetabaseState['parameterValues']
+  ) {
   // Algo:
   // transitivity is: template-tags -> dashcard parameters -> dashcard parameter mappings -> dashboard parameters -> parameter values
   //                                        |-> parameter values
@@ -268,9 +271,9 @@ export async function getDashboardAppState(): Promise<MetabaseAppStateDashboard
   }
   const selectedTabDashcardIds = getSelectedTabDashcardIds(dashboardMetabaseState);
 //   const dashboardParameters = _.get(dashboardMetabaseState, ['dashboards', dashboardId, 'parameters'], [])
-  const cards = await Promise.all(selectedTabDashcardIds.map(async dashcardId => await getDashcardInfoWithSQLAndOutputTableMd(dashboardMetabaseState, dashcardId, dashboardId)))
-
-  const filteredCards = _.compact(cards);
+  const allModels = dbId ?  await getAllRelevantModelsForSelectedDb(dbId) : []
+  const cards = await Promise.all(selectedTabDashcardIds.map(async dashcardId => await getDashcardInfoWithSQLAndOutputTableMd(dashboardMetabaseState, dashcardId, dashboardId, allModels)))
+  let filteredCards = _.compact(cards);
   let sqlTables: TableAndSchema[] = []
   forEach(filteredCards, (card) => {
     if (card) {
@@ -284,9 +287,26 @@ export async function getDashboardAppState(): Promise<MetabaseAppStateDashboard
       })
     }
   })
+
+
   sqlTables = _.uniqBy(sqlTables, (table) => `${table.schema}::${table.name}`)
   const relevantTablesWithFields = await getTablesWithFields(appSettings.tableDiff, appSettings.drMode, !!selectedCatalog, sqlTables, [])
-  const tableContextYAML = getTableContextYAML(relevantTablesWithFields, selectedCatalog, appSettings.drMode);
+  // find a list of models from each native card using getModelsFromSql, and then merge them to get relevantModels
+  const modelsFromAllCards = (await Promise.all(filteredCards.map(async card => {
+    if (card.sql) {
+      return await getModelsFromSql(card.sql, allModels)
+    }
+    return []
+  }))).flat()
+  const dedupedCardAndSelectedModels = _.uniqBy([...modelsFromAllCards, ...appSettings.selectedModels], 'modelId')
+  const relevantModelsWithFields = await getModelsWithFields(dedupedCardAndSelectedModels)
+  const allFormattedTables = [...relevantTablesWithFields, ...relevantModelsWithFields]
+  const tableContextYAML = getTableContextYAML(allFormattedTables, selectedCatalog, appSettings.drMode);
+  filteredCards = filteredCards.map(card => {
+    // replace model identifiers with model ids
+    card.sql = modifySqlForMetabaseModels(card.sql, allModels)
+    return card
+  })
   dashboardInfo.cards = filteredCards
   // filter out dashcards with null names or ids
   .filter(dashcard => dashcard.name !== null && dashcard.id !== null);

apps/src/metabase/helpers/metabaseModels.ts

@@ -104,7 +104,7 @@ export const getModelsWithFields = async (models: MetabaseModel[]) => {
 
 // get any models in the sql that look like {{#1234-some-model-name}} 
 // verify that the model with that id exists in all models
-const getModelsFromSql = async (sql: string, allModels: MetabaseModel[]) => {
+export const getModelsFromSql = async (sql: string, allModels: MetabaseModel[]) => {
     const regex = /{{#(\d+)-.*?}}/g;
     const matches = [...sql.matchAll(regex)];
     const modelIds = matches.map(match => match[1])

apps/src/package.ts

@@ -3,5 +3,5 @@ export { applyTableDiffs } from "./common/utils";
 export { getTableContextYAML, filterTablesByCatalog } from "./metabase/helpers/catalog";
 export { getTableData } from "./metabase/helpers/metabaseAPIHelpers";
 export { getAllTemplateTagsInQuery } from "./metabase/helpers/sqlQuery";
-export { getModelsWithFields, getSelectedAndRelevantModels } from "./metabase/helpers/metabaseModels";
+export { getModelsWithFields, getSelectedAndRelevantModels, modifySqlForMetabaseModels, replaceLLMFriendlyIdentifiersInSqlWithModels } from "./metabase/helpers/metabaseModels";
 export { getCurrentQuery } from "./metabase/helpers/metabaseStateAPI";

web/src/components/common/Markdown.tsx

@@ -10,7 +10,8 @@ import { renderString } from '../../helpers/templatize'
 import { getOrigin } from '../../helpers/origin'
 import { getApp } from '../../helpers/app'
 import { processSQLWithCtesOrModels } from '../../helpers/catalogAsModels'
-import { getAllTemplateTagsInQuery } from 'apps'
+import { getAllTemplateTagsInQuery, replaceLLMFriendlyIdentifiersInSqlWithModels } from 'apps'
+import type { MetabaseModel } from 'apps/types'
 import { Badge } from "@chakra-ui/react";
 
 
@@ -159,9 +160,13 @@ export function Markdown({content, messageIndex}: {content: string, messageIndex
     if (content.includes('{{MX_LAST_SQL_URL}}')) {
       try {
         // Extract last SQL from messages before the current message
-        const lastSQL = messageIndex !== undefined 
+        let lastSQL = messageIndex !== undefined 
           ? extractLastSQLFromMessages(currentThread?.messages || [], messageIndex)
           : null;
+        if (lastSQL) {
+          const allModels: MetabaseModel[] = toolContext?.dbInfo?.models || []
+          lastSQL = replaceLLMFriendlyIdentifiersInSqlWithModels(lastSQL, allModels)
+        }
 
         if (lastSQL) {
           // Get Metabase origin from iframe info