Dashboard explorer (#298)

* let dashcard be the base card, handle limit entities

* add parametervalues

* link render bug fix

* new tool in app controller

* add mbql via dataset

* add continue analysis

* Specify if no data returned in query execution

* Continue MBQL threads from dashboard

* Allow user to continue old threads

---------

Co-authored-by: nuwandavek <vivekaithal44@gmail.com>

Author: ppsreejith

apps/src/base/appController.ts

@@ -4,7 +4,7 @@ import { RPCs, utils } from "web";
 import { ActionRenderInfo, DefaultMessageContent } from "web/types"
 import 'reflect-metadata';
 
-interface App<T> {
+export interface App<T> {
   getState: () => Promise<T>;
   getQuerySelectorMap: () => Promise<QuerySelectorMap>;
 }

apps/src/metabase/appController.ts

@@ -7,9 +7,10 @@ import {
   MetabaseAppStateSQLEditor,
   MetabaseSemanticQueryAppState,
   MetabaseAppStateMBQLEditor,
-  MetabasePageType,
 } from "./helpers/DOMToState";
 
+import { MetabasePageType } from "./helpers/utils";
+
 import { MetabaseAppStateSQLEditorV2 } from "./helpers/analystModeTypes";
 
 import {
@@ -45,7 +46,7 @@ import {
 } from "./helpers/sqlQuery";
 import axios from 'axios'
 import { getSelectedDbId, getCurrentUserInfo as getUserInfo, getSnippets, getCurrentCard, getDashboardState, getCurrentQuery, getParameterValues } from "./helpers/metabaseStateAPI";
-import { runSQLQueryFromDashboard } from "./helpers/dashboard/runSqlQueryFromDashboard";
+import { runSQLQueryFromDashboard, runMBQLQueryFromDashboard } from "./helpers/dashboard/runSqlQueryFromDashboard";
 import { getAllRelevantModelsForSelectedDb, getTableData } from "./helpers/metabaseAPIHelpers";
 import { processSQLWithCtesOrModels, dispatch, updateIsDevToolsOpen, updateDevToolsTabName, addMemory } from "web";
 import { fetchTableMetadata } from "./helpers/metabaseAPI";
@@ -266,6 +267,20 @@ export class MetabaseController extends AppController<MetabaseAppState> {
     return actionContent;
   }
 
+  async runMBQLQuery({ mbql, dbID }: { mbql: any, dbID: number }) {
+    const actionContent: BlankMessageContent = {
+      type: "BLANK",
+    };
+    const response = await runMBQLQueryFromDashboard(mbql, dbID);
+    if (response.error) {
+      actionContent.content = `<ERROR>${response.error}</ERROR>`;
+    } else {
+      const asMarkdown = metabaseToCSV(response.data);
+      actionContent.content = asMarkdown;
+    }
+    return actionContent;
+  }
+
   @Action({
     labelRunning: "Running SQL Query with parameters",
     labelDone: "Ran SQL query with parameters",
@@ -560,6 +575,64 @@ export class MetabaseController extends AppController<MetabaseAppState> {
     return actionContent;
   }
 
+  @Action({
+    labelRunning: "Constructs the MBQL query",
+    labelDone: "MBQL built",
+    labelTask: "Built MBQL query",
+    description: "Constructs the MBQL query in the GUI editor",
+    renderBody: ({ mbql, explanation }: { mbql: any, explanation: string }) => {
+        if (isEmpty(mbql)) {
+            return {text: "This MBQL query has errors", code: null, language: "markdown"}
+        }
+      return {text: explanation, code: JSON.stringify(mbql), language: "json"}
+    }
+  })
+  async ExecuteMBQLQuery({ mbql, explanation }: { mbql: any, explanation: string }) {
+    const actionContent: BlankMessageContent = {
+        type: "BLANK",
+    };
+    const state = (await this.app.getState()) as MetabaseAppStateMBQLEditor;
+    const dbID = state?.selectedDatabaseInfo?.id as number
+    if (!dbID) {
+      actionContent.content = "No database selected";
+      return actionContent;
+    }
+    if (isEmpty(mbql)) {
+        actionContent.content = "This MBQL query has errors: " + explanation;
+        return actionContent;
+    }
+
+    if (mbql) {
+        const table_ids = getSourceTableIds(mbql);
+        await updateMBEntities(table_ids)
+    }
+
+    const finCard = {
+        type: "question",
+        visualization_settings: {},
+        display: "table",
+        dataset_query: {
+            database: dbID,
+            type: "query",
+            query: mbql
+        }
+    };
+
+    const metabaseState = this.app as App<MetabaseAppState>;
+    const pageType = metabaseState.useStore().getState().toolContext?.pageType as MetabasePageType;
+    if (pageType === 'mbql') {
+      // # Ensure you're in mbql editor mode
+      await RPCs.dispatchMetabaseAction('metabase/qb/SET_UI_CONTROLS', {
+        queryBuilderMode: "notebook",
+      });
+      await RPCs.dispatchMetabaseAction('metabase/qb/UPDATE_QUESTION', {card: finCard});
+      return await this._executeMBQLQueryInternal()
+    }
+    else if ((pageType === 'dashboard') || (pageType === 'unknown')) {
+        return await this.runMBQLQuery({mbql, dbID});
+    }
+  }
+
   @Action({
     labelRunning: "Constructs the MBQL query",
     labelDone: "MBQL built",

apps/src/metabase/helpers/DOMToState.ts

@@ -19,7 +19,7 @@ import { MBQLInfo } from './mbql/utils';
 import { getModelsWithFields, getSelectedAndRelevantModels, modifySqlForMetabaseModels} from './metabaseModels';
 import { MetabaseAppStateSQLEditorV2, MetabaseAppStateType, processCard } from './analystModeTypes';
 import { MetabaseTableOrModel } from './metabaseAPITypes';
-import { determineMetabasePageType, MetabasePageType } from './utils';
+import { determineMetabasePageType, MetabasePageType, getLimitedEntities } from './utils';
 
 const {modifySqlForMxModels} = catalogAsModels
 
@@ -80,6 +80,7 @@ export interface MetabaseAppStateDashboard extends DashboardInfo {
   metabaseUrl?: string;
   isEmbedded: boolean;
   limitedEntities?: MetabaseTableOrModel[];
+  parameterValues?: ParameterValues;
 }
 
 export interface MetabaseAppStateMBQLEditor extends MBQLInfo {
@@ -105,62 +106,6 @@ export interface MetabaseSemanticQueryAppState {
 export type MetabaseAppState = MetabaseAppStateSQLEditor | MetabaseAppStateDashboard | MetabaseSemanticQueryAppState | MetabaseAppStateMBQLEditor | MetabaseAppStateSQLEditorV2;
 
 // no need to fetch fields since we don't want that in limited entities
-async function getLimitedEntities(sqlQuery: string): Promise<MetabaseTableOrModel[]> {
-  const appSettings = RPCs.getAppSettings();
-  
-  // Early return if conditions not met
-  if (!appSettings.analystMode || !appSettings.manuallyLimitContext) {
-    return [];
-  }
-  
-  const dbId = await getSelectedDbId();
-  const selectedDatabaseInfo = dbId ? await getDatabaseInfo(dbId) : undefined;
-  const defaultSchema = selectedDatabaseInfo?.default_schema;
-  
-  const sqlTables = getTablesFromSqlRegex(sqlQuery);
-  const selectedCatalogObj = find(appSettings.availableCatalogs, { name: appSettings.selectedCatalog });
-  const selectedCatalog = get(selectedCatalogObj, 'content');
-  
-  // Apply default schema to tables if needed
-  if (defaultSchema) {
-    sqlTables.forEach((table) => {
-      if (table.schema === undefined || table.schema === '') {
-        table.schema = defaultSchema;
-      }
-    });
-  }
-  
-  let relevantTablesWithFields = await getTablesWithFields(appSettings.tableDiff, appSettings.drMode, !!selectedCatalog, sqlTables, []);
-  
-  // Add defaultSchema back to relevantTablesWithFields
-  relevantTablesWithFields = relevantTablesWithFields.map(table => {
-    if (table.schema === undefined || table.schema === '') {
-      table.schema = defaultSchema || 'unknown';
-    }
-    return table;
-  });
-  
-  const allModels = dbId ? await getAllRelevantModelsForSelectedDb(dbId) : [];
-  const relevantModels = await getSelectedAndRelevantModels(sqlQuery || "", appSettings.selectedModels, allModels);
-  
-  // Transform and combine tables and models with type annotations
-  const relevantTablesWithFieldsAndType: MetabaseTableOrModel[] = relevantTablesWithFields.map(table => ({
-    type: 'table',
-    id: table.id,
-    name: table.name,
-    schema: table.schema,
-    description: table.description,
-  }));
-  
-  const relevantModelsWithFieldsAndType: MetabaseTableOrModel[] = relevantModels.map(model => ({
-    type: 'model',
-    id: model.modelId || 0,
-    name: model.name,
-    description: model.description,
-  }));
-  
-  return [...relevantTablesWithFieldsAndType, ...relevantModelsWithFieldsAndType];
-}
 
 export async function convertDOMtoStateSQLQueryV2(pageType: MetabasePageType) : Promise<MetabaseAppStateSQLEditorV2> {
   const [metabaseUrl, currentCardRaw, outputMarkdown, parameterValues] = await Promise.all([

apps/src/metabase/helpers/dashboard/appState.ts

@@ -13,6 +13,9 @@ import { getTableContextYAML } from '../catalog';
 import { getModelsFromSql, getModelsWithFields, modifySqlForMetabaseModels, replaceLLMFriendlyIdentifiersInSqlWithModels } from '../metabaseModels';
 import { MetabaseAppStateType } from '../analystModeTypes';
 import { MetabaseTableOrModel } from '../metabaseAPITypes';
+import { processCard } from '../analystModeTypes';
+import { Card, SavedCard } from '../types';
+import { getLimitedEntitiesFromQueries, getLimitedEntitiesFromMBQLQueries } from '../utils';
 
 // Removed: const { getMetabaseState } = RPCs - using centralized state functions instead
 
@@ -77,14 +80,7 @@ export type DashboardInfoForModelling = {
     type: string,
     value?: string | null
   }[];
-  cards: {
-    id: number,
-    name: string,
-    sql: string,
-    databaseId: number,
-    description?: string | undefined,
-    outputTableMarkdown?: string,
-  }[]
+  cards: SavedCard[]
 }
 
 function substituteParameterMappings(
@@ -103,65 +99,30 @@ function substituteParameterMappings(
   return sql
 }
 
-async function getDashcardInfoWithSQLAndOutputTableMd(
+async function getDashcardwithOutputTableMd(
   dashboardMetabaseState: DashboardMetabaseState, 
   dashcardId: number,
   dashboardId: number): Promise<DashboardInfoForModelling['cards'][number] | null> {
-  const dashcard = dashboardMetabaseState.dashcards[dashcardId];
+  const dashcard = dashboardMetabaseState.dashcards[dashcardId].card;
   if (!dashcard) {
     return null;
   }
-  const cardId = _.get(dashcard, 'card_id', '');
-  const databaseId = _.get(dashcard, 'card.database_id', 0);
-  const id = _.get(dashcard, 'id');
-  const query_type = _.get(dashcard, 'card.query_type', 'unknown');
-  const name = _.get(dashcard, 'card.name', '');
-  const description = _.get(dashcard, 'card.description', '');
-  const visualizationType = _.get(dashcard, 'card.display', '');
-  if (!name)
+  const cardID = _.get(dashcard, 'id', null);
+  if (!cardID) {
     return null;
-  let sql = ''
-  if (query_type == 'native'){
-    sql = _.get(dashcard, 'card.dataset_query.native.query', '');
-  }
-  else if (query_type == 'query'){
-    // mbql query
-    const mbqlQuery = _.get(dashcard, 'card.dataset_query.query', {});
-    if (mbqlQuery) {
-      const sql_from_mbql = await getSQLFromMBQL({
-        database: databaseId,
-        type: 'query',
-        query: mbqlQuery
-      });
-      sql = sql_from_mbql.query || '';
-    }
   }
-  
-  if (!sql || sql == '')
-    return null;
+  const card = processCard(dashcard) as SavedCard;
+  card.id = cardID;
 
-  // replace parameters
-  try {
-    sql = await substituteParameters(sql, dashcard, dashboardMetabaseState['dashboards'][dashboardId]?.param_fields, dashboardMetabaseState.parameterValues)
-  } catch (e) {
-  }
-  const obj = {
-    id,
-    name,
-    sql,
-    databaseId,
-    visualizationType,
-    ...(description ? { description } : {}),
-  }
   // dashcardData
-  const data = _.get(dashboardMetabaseState, ['dashcardData', dashcardId, cardId, 'data']);
+  const data = _.get(dashboardMetabaseState, ['dashcardData', dashcardId, card.id, 'data']);
   if (!data) {
-    return obj
+    return card
   }
   const dataAsMarkdown = metabaseToMarkdownTable(data, 1000);
   return {
-   ...obj,
-   outputTableMarkdown: dataAsMarkdown
+    ...card,
+    outputTableMarkdown: dataAsMarkdown
   }
 }
 /* 
@@ -278,75 +239,33 @@ export async function getDashboardAppState(): Promise<MetabaseAppStateDashboard
   }
   const selectedTabDashcardIds = getSelectedTabDashcardIds(dashboardMetabaseState);
 //   const dashboardParameters = _.get(dashboardMetabaseState, ['dashboards', dashboardId, 'parameters'], [])
-  const allModels = dbId ?  await getAllRelevantModelsForSelectedDb(dbId) : []
-  const cards = await Promise.all(selectedTabDashcardIds.map(async dashcardId => await getDashcardInfoWithSQLAndOutputTableMd(dashboardMetabaseState, dashcardId, dashboardId, allModels)))
+  const cards = await Promise.all(selectedTabDashcardIds.map(async dashcardId => await getDashcardwithOutputTableMd(dashboardMetabaseState, dashcardId, dashboardId)))
   let filteredCards = _.compact(cards);
-  let sqlTables: TableAndSchema[] = []
-  forEach(filteredCards, (card) => {
-    if (card) {
-      getTablesFromSqlRegex(card.sql).forEach((table) => {
-        if (defaultSchema) {
-          if (table.schema === undefined || table.schema === '') {
-            table.schema = defaultSchema
-          }
-        }
-        sqlTables.push(table)
-      })
-    }
-  })
-
-
-  sqlTables = _.uniqBy(sqlTables, (table) => `${table.schema}::${table.name}`)
-  const relevantTablesWithFields = await getTablesWithFields(appSettings.tableDiff, appSettings.drMode, !!selectedCatalog, sqlTables, [])
-  // find a list of models from each native card using getModelsFromSql, and then merge them to get relevantModels
-  const modelsFromAllCards = (await Promise.all(filteredCards.map(async card => {
-    if (card.sql) {
-      return await getModelsFromSql(card.sql, allModels)
-    }
-    return []
-  }))).flat()
-  const dedupedCardAndSelectedModels = _.uniqBy([...modelsFromAllCards, ...appSettings.selectedModels], 'modelId')
-  const relevantModelsWithFields = await getModelsWithFields(dedupedCardAndSelectedModels)
-  const allFormattedTables = [...relevantTablesWithFields, ...relevantModelsWithFields]
-  const tableContextYAML = getTableContextYAML(allFormattedTables, selectedCatalog, appSettings.drMode);
-  filteredCards = filteredCards.map(card => {
-    // replace model identifiers with model ids
-    card.sql = modifySqlForMetabaseModels(card.sql, allModels)
-    return card
-  })
-  dashboardInfo.cards = filteredCards
-  // filter out dashcards with null names or ids
-  .filter(dashcard => dashcard.name !== null && dashcard.id !== null);
-  // remove description if it's null or undefined
-  if (!dashboardInfo.description) {
-    delete dashboardInfo.description;
-  }
+    const limitedEntitiesSQL = await getLimitedEntitiesFromQueries(
+        filteredCards.flatMap(card => 
+            card?.dataset_query?.native?.query ? [card.dataset_query.native.query] : []
+        )
+    );
+    const limitedEntitiesMBQL = await getLimitedEntitiesFromMBQLQueries(
+        filteredCards.flatMap(card => 
+            card?.dataset_query?.query ? [card.dataset_query.query] : []
+        )
+    );
+    const limitedEntities = [...limitedEntitiesSQL, ...limitedEntitiesMBQL];
+    // remove duplicates based on id and type
+    const uniqueEntities = Array.from(new Map(limitedEntities.map(entity => [entity.id, entity])).values());
   const dashboardAppState: MetabaseAppStateDashboard = {
     ...dashboardInfo,
     type: MetabaseAppStateType.Dashboard,
-    tableContextYAML,
     selectedDatabaseInfo,
     metabaseOrigin: url,
     metabaseUrl: fullUrl,
     isEmbedded: getParsedIframeInfo().isEmbedded,
   };
-  if (appSettings.analystMode && appSettings.manuallyLimitContext) {
-    const limitToTables: MetabaseTableOrModel[] = relevantTablesWithFields.map(table => ({
-      type: 'table',
-      id: table.id,
-      name: table.name,
-      schema: table.schema,
-      description: table.description,
-    }))
-    const limitToModels: MetabaseTableOrModel[] = dedupedCardAndSelectedModels.map(model => ({
-      type: 'model',
-      id: model.modelId,
-      name: model.name,
-      description: model.description,
-    }))
-    dashboardAppState.limitedEntities = [...limitToTables, ...limitToModels];
-  }
-  return dashboardAppState
+  dashboardAppState.cards = filteredCards as SavedCard[];
+  dashboardAppState.limitedEntities = uniqueEntities;
+  dashboardAppState.parameterValues = dashboardMetabaseState.parameterValues || {};
+  return dashboardAppState;
 }