feat: add support for configurable views for child databases

Author: JohannesRudolph

src/BlockRenderer.ts

@@ -1,22 +1,12 @@
 import {
-  Block as PublicBlock,
-  BlockBase,
-  Emoji,
-  File,
-  ExternalFile,
-  ExternalFileWithCaption,
-  FileWithCaption,
-  ImageBlock,
-  RichText,
-} from "@notionhq/client/build/src/api-types";
-
-import * as markdownTable from "./markdown-table";
-import { AssetWriter } from "./AssetWriter";
-import { Database } from "./Database";
-import { DeferredRenderer } from "./DeferredRenderer";
-import { RichTextRenderer } from "./RichTextRenderer";
-import { logger } from "./logger";
-import { LinkRenderer } from "./LinkRenderer";
+    Block as PublicBlock, BlockBase, Emoji, ExternalFile, ExternalFileWithCaption, File,
+    FileWithCaption, ImageBlock, RichText
+} from '@notionhq/client/build/src/api-types';
+
+import { AssetWriter } from './AssetWriter';
+import { DeferredRenderer } from './DeferredRenderer';
+import { logger } from './logger';
+import { RichTextRenderer } from './RichTextRenderer';
 
 const debug = require("debug")("blocks");
 
@@ -65,9 +55,8 @@ export type Block =
 
 export class BlockRenderer {
   constructor(
-    private readonly deferredRenderer: DeferredRenderer,
     private readonly richText: RichTextRenderer,
-    private readonly linkRenderer: LinkRenderer
+    private readonly deferredRenderer: DeferredRenderer,
   ) {}
 
   async renderBlock(block: Block, assets: AssetWriter): Promise<string> {
@@ -79,22 +68,26 @@ export class BlockRenderer {
       case "heading_2":
         return "## " + await this.richText.renderMarkdown(block.heading_2.text);
       case "heading_3":
-        return "### " + await this.richText.renderMarkdown(block.heading_3.text);
+        return "### " +
+          await this.richText.renderMarkdown(block.heading_3.text);
       case "bulleted_list_item":
         return (
-          "- " + await this.richText.renderMarkdown(block.bulleted_list_item.text)
+          "- " +
+          await this.richText.renderMarkdown(block.bulleted_list_item.text)
         );
       case "numbered_list_item":
         return (
-          "1. " + await this.richText.renderMarkdown(block.numbered_list_item.text)
+          "1. " +
+          await this.richText.renderMarkdown(block.numbered_list_item.text)
         );
       case "to_do":
         return "[ ] " + await this.richText.renderMarkdown(block.to_do.text);
       case "image":
         return await this.renderImage(block, assets);
       case "quote":
         block as any;
-        return "> " + await this.richText.renderMarkdown((block as any).quote.text);
+        return "> " +
+          await this.richText.renderMarkdown((block as any).quote.text);
       case "code":
         return (
           "```" +
@@ -113,17 +106,7 @@ export class BlockRenderer {
       case "divider":
         return "---";
       case "child_database":
-        // queue all pages in the database for individual, deferred rendering
-        const db = await this.deferredRenderer.renderDatabasePages(block.id);
-        const msg = `<!-- included database ${block.id} -->\n`;
-
-        if (db.config.skipMarkdownTable) {
-          return msg;
-        }
-
-        // todo: make this nicer, e.g. render multi tables
-        return msg + this.renderTables(db);
-
+        return await this.deferredRenderer.renderChildDatabase(block.id);
       case "toggle":
       case "child_page":
       case "embed":
@@ -136,35 +119,11 @@ export class BlockRenderer {
       default:
         return this.renderUnsupported(
           `unsupported block type: ${block.type}`,
-          block
+          block,
         );
     }
   }
 
-  renderTables(db: Database) {
-    // todo: handle empty page
-    const props = db.pages[0].properties;
-
-    const table: any[][] = [];
-
-    const headers = Array.from(props.keys.keys());
-    table[0] = headers;
-
-    const cols = Array.from(props.keys.values());
-    db.pages.forEach((r) =>
-      table.push(
-        cols.map((c, i) => {
-          const content = escapeTableCell(r.properties.values[c]);
-          return i == 0
-            ? this.linkRenderer.renderPageLink(content, r) // make the first cell a relative link to the page
-            : content;
-        })
-      )
-    );
-
-    return markdownTable.markdownTable(table);
-  }
-
   private renderIcon(icon: File | ExternalFile | Emoji): string {
     switch (icon.type) {
       case "emoji":
@@ -173,7 +132,7 @@ export class BlockRenderer {
       case "external":
         return this.renderUnsupported(
           `unsupported icon type: ${icon.type}`,
-          icon
+          icon,
         );
     }
   }
@@ -204,12 +163,3 @@ export class BlockRenderer {
     return `<!-- ${msg} -->`;
   }
 }
-
-function escapeTableCell(content: string | number | any): string {
-  // markdown table cells do not support newlines, however we can insert <br> elements instead
-  if (typeof content === "string") {
-    return content.replace(/\n/g, "<br>");
-  }
-
-  return content.toString();
-}

src/ChildDatabaseRenderer.ts

@@ -0,0 +1,71 @@
+import { Client } from '@notionhq/client/build/src';
+import { Page } from '@notionhq/client/build/src/api-types';
+
+import { SyncConfig } from './';
+import { lookupDatabaseConfig } from './config';
+import { DatabaseViewRenderer } from './DatabaseViewRenderer';
+import { DeferredRenderer } from './DeferredRenderer';
+import { DatabaseConfig } from './SyncConfig';
+import { TableRenderer } from './TableRenderer';
+
+export class ChildDatabaseRenderer {
+  constructor(
+    private readonly config: SyncConfig,
+    private readonly publicApi: Client,
+    private readonly deferredRenderer: DeferredRenderer,
+    private readonly tableRenderer: TableRenderer,
+    private readonly viewRenderer: DatabaseViewRenderer,
+  ) {}
+
+  async renderChildDatabase(databaseId: string): Promise<string> {
+    const dbConfig = lookupDatabaseConfig(this.config, databaseId);
+
+    const msg = `<!-- included database ${databaseId} -->\n`;
+
+    // no view was defined for this database, render as a plain inline table
+    const allPages = await this.fetchPages(databaseId, dbConfig);
+
+    const isCmsDb = this.config.cmsDatabaseId !== databaseId;
+    if (isCmsDb && !dbConfig.views) {
+      return msg + await this.tableRenderer.renderTable(allPages, dbConfig);
+    }
+
+    // queue all pages in the database for individual, deferred rendering
+    const prepareRenderPageTasks = allPages.map((x) =>
+      this.deferredRenderer.renderPage(x, dbConfig)
+    );
+
+    // note: the await here is not actually starting to render the pages, however it prepares the page render task
+    const renderPageTasks = await Promise.all(prepareRenderPageTasks);
+
+    const db = {
+      pages: renderPageTasks,
+      config: dbConfig,
+    };
+
+    return this.viewRenderer.renderViews(db);
+  }
+
+  private async fetchPages(
+    databaseId: string,
+    dbConfig: DatabaseConfig,
+  ): Promise<Page[]> {
+    const db = await this.publicApi.databases.retrieve({
+      database_id: databaseId,
+    });
+
+    const allPages = await this.publicApi.databases.query({
+      database_id: db.id,
+      sorts: dbConfig.sorts,
+      page_size: 100,
+    }); // todo: paging
+
+    if (allPages.next_cursor) {
+      throw new Error(
+        `Paging not implemented, db ${db.id} has more than 100 entries`,
+      );
+    }
+
+    return allPages.results;
+  }
+}

src/DatabaseRenderer.ts

@@ -0,0 +1,62 @@
+import { Database } from './Database';
+import { TableRenderer } from './TableRenderer';
+
+const debug = require("debug")("database");
+
+export class DatabaseViewRenderer {
+  constructor(
+    private readonly tableRenderer: TableRenderer,
+  ) {}
+
+  renderViews(db: Database): string {
+    const views = db.config.views?.map((view) => {
+      const propKeys = db.pages[0].properties.keys;
+      const propKey = propKeys.get(view.properties.groupBy);
+
+      if (!propKey) {
+        const msg =
+          `Could not render view ${view.title}, groupBy property ${view.properties.groupBy} not found`;
+        debug(msg + "%O", view);
+        throw new Error(msg);
+      }
+
+      const grouped = new Array(
+        ...groupBy(db.pages, (p) => p.properties.values[propKey]),
+      );
+
+      return grouped
+        .map(([key, pages]) => this.tableRenderer.renderView(pages, key, view))
+        .join("\n\n");
+    });
+
+    return views?.join("\n\n") || "";
+  }
+}
+
+/**
+ * @description
+ * Takes an Array<V>, and a grouping function,
+ * and returns a Map of the array grouped by the grouping function.
+ *
+ * @param list An array of type V.
+ * @param keyGetter A Function that takes the the Array type V as an input, and returns a value of type K.
+ *                  K is generally intended to be a property key of V.
+ *
+ * @returns Map of the array grouped by the grouping function.
+ */
+export function groupBy<K, V>(
+  list: Array<V>,
+  keyGetter: (input: V) => K,
+): Map<K, Array<V>> {
+  const map = new Map<K, Array<V>>();
+  list.forEach((item) => {
+    const key = keyGetter(item);
+    const collection = map.get(key);
+    if (!collection) {
+      map.set(key, [item]);
+    } else {
+      collection.push(item);
+    }
+  });
+  return map;
+}

src/DatabaseViewRenderer.ts

@@ -1,41 +1,33 @@
-import { Page } from "@notionhq/client/build/src/api-types";
-import { DatabaseConfig } from "./SyncConfig";
-import { Database } from "./Database";
-import { DatabaseRenderer } from "./DatabaseRenderer";
-import { logger } from "./logger";
-import { PageRenderer } from "./PageRenderer";
-import { RenderPageTask as RenderPageTask } from "./RenderPageTask";
-import { RenderedPage } from "./RenderedPage";
+import { Page } from '@notionhq/client/build/src/api-types';
+
+import { ChildDatabaseRenderer } from './ChildDatabaseRenderer';
+import { logger } from './logger';
+import { PageRenderer } from './PageRenderer';
+import { RenderedPage } from './RenderedPage';
+import { RenderPageTask as RenderPageTask } from './RenderPageTask';
+import { DatabaseConfig } from './SyncConfig';
 
 const debug = require("debug")("rendering");
 
 export class DeferredRenderer {
-  private dbRenderer!: DatabaseRenderer;
+  private dbRenderer!: ChildDatabaseRenderer;
   private pageRenderer!: PageRenderer;
 
   private pageQueue: (() => Promise<any>)[] = [];
 
-  private readonly renderedDatabases = new Map<string, Database>();
   private readonly renderedPages = new Map<string, RenderPageTask>();
 
   public initialize(
-    dbRenderer: DatabaseRenderer,
+    dbRenderer: ChildDatabaseRenderer,
     pageRenderer: PageRenderer
   ) {
     this.dbRenderer = dbRenderer;
     this.pageRenderer = pageRenderer;
   }
 
-  public async renderDatabasePages(databaseId: string): Promise<Database> {
-    const cached = this.renderedDatabases.get(databaseId);
-    if (cached) {
-      debug("db cache hit " + databaseId);
-      return cached;
-    }
-
+  public async renderChildDatabase(databaseId: string): Promise<string> {
     // database pages objects are retrieved immediately, but page bodys are queued for deferred rendering
-    const fetched = await this.dbRenderer.renderDatabase(databaseId);
-    this.renderedDatabases.set(databaseId, fetched);
+    const fetched = await this.dbRenderer.renderChildDatabase(databaseId);
 
     return fetched;
   }