feat: render database entries to index

refactor to make rendering paths between pages and entries symmetrical

Author: JohannesRudolph

src/BlockRenderer.ts

@@ -1,12 +1,19 @@
 import {
-    Block as PublicBlock, BlockBase, Emoji, ExternalFile, ExternalFileWithCaption, File,
-    FileWithCaption, ImageBlock, RichText
-} from '@notionhq/client/build/src/api-types';
-
-import { AssetWriter } from './AssetWriter';
-import { DeferredRenderer } from './DeferredRenderer';
-import { logger } from './logger';
-import { RichTextRenderer } from './RichTextRenderer';
+  Block as PublicBlock,
+  BlockBase,
+  Emoji,
+  ExternalFile,
+  ExternalFileWithCaption,
+  File,
+  FileWithCaption,
+  ImageBlock,
+  RichText,
+} from "@notionhq/client/build/src/api-types";
+
+import { AssetWriter } from "./AssetWriter";
+import { DeferredRenderer } from "./DeferredRenderer";
+import { logger } from "./logger";
+import { RichTextRenderer } from "./RichTextRenderer";
 
 const debug = require("debug")("blocks");
 
@@ -56,7 +63,7 @@ export type Block =
 export class BlockRenderer {
   constructor(
     private readonly richText: RichTextRenderer,
-    private readonly deferredRenderer: DeferredRenderer,
+    private readonly deferredRenderer: DeferredRenderer
   ) {}
 
   async renderBlock(block: Block, assets: AssetWriter): Promise<string> {
@@ -65,49 +72,57 @@ export class BlockRenderer {
         return await this.richText.renderMarkdown(block.paragraph.text);
       // note: render headings +1 level, because h1 is reserved for page titles
       case "heading_1":
-        return "## " + await this.richText.renderMarkdown(block.heading_1.text);
+        return (
+          "## " + (await this.richText.renderMarkdown(block.heading_1.text))
+        );
       case "heading_2":
-        return "### " + await this.richText.renderMarkdown(block.heading_2.text);
+        return (
+          "### " + (await this.richText.renderMarkdown(block.heading_2.text))
+        );
       case "heading_3":
-        return "#### " +
-          await this.richText.renderMarkdown(block.heading_3.text);
+        return (
+          "#### " + (await this.richText.renderMarkdown(block.heading_3.text))
+        );
       case "bulleted_list_item":
         return (
           "- " +
-          await this.richText.renderMarkdown(block.bulleted_list_item.text)
+          (await this.richText.renderMarkdown(block.bulleted_list_item.text))
         );
       case "numbered_list_item":
         return (
           "1. " +
-          await this.richText.renderMarkdown(block.numbered_list_item.text)
+          (await this.richText.renderMarkdown(block.numbered_list_item.text))
         );
       case "to_do":
-        return "[ ] " + await this.richText.renderMarkdown(block.to_do.text);
+        return "[ ] " + (await this.richText.renderMarkdown(block.to_do.text));
       case "image":
         return await this.renderImage(block, assets);
       case "quote":
         block as any;
-        return "> " +
-          await this.richText.renderMarkdown((block as any).quote.text);
+        return (
+          "> " + (await this.richText.renderMarkdown((block as any).quote.text))
+        );
       case "code":
         return (
           "```" +
           block.code.language +
           "\n" +
-          await this.richText.renderMarkdown(block.code.text) +
+          (await this.richText.renderMarkdown(block.code.text)) +
           "\n```"
         );
       case "callout":
         return (
           "> " +
           this.renderIcon(block.callout.icon) +
           " " +
-          await this.richText.renderMarkdown(block.callout.text)
+          (await this.richText.renderMarkdown(block.callout.text))
         );
       case "divider":
         return "---";
       case "child_database":
-        return await this.deferredRenderer.renderChildDatabase(block.id);
+        const msg = `<!-- included database ${block.id} -->\n`;
+        const db = await this.deferredRenderer.renderChildDatabase(block.id);
+        return msg + db.markdown;
       case "toggle":
       case "child_page":
       case "embed":
@@ -120,7 +135,7 @@ export class BlockRenderer {
       default:
         return this.renderUnsupported(
           `unsupported block type: ${block.type}`,
-          block,
+          block
         );
     }
   }
@@ -133,7 +148,7 @@ export class BlockRenderer {
       case "external":
         return this.renderUnsupported(
           `unsupported icon type: ${icon.type}`,
-          icon,
+          icon
         );
     }
   }

src/ChildDatabaseRenderer.ts

@@ -1,54 +1,81 @@
-import { Client } from '@notionhq/client/build/src';
-import { Page } from '@notionhq/client/build/src/api-types';
+import { Client } from "@notionhq/client/build/src";
+import { Page } from "@notionhq/client/build/src/api-types";
 
-import { SyncConfig } from './';
-import { lookupDatabaseConfig } from './config';
-import { DatabaseViewRenderer } from './DatabaseViewRenderer';
-import { DeferredRenderer } from './DeferredRenderer';
-import { DatabaseConfig } from './SyncConfig';
-import { TableRenderer } from './TableRenderer';
+import { SyncConfig } from "./";
+import { lookupDatabaseConfig } from "./config";
+import { Database } from "./Database";
+import { DatabaseViewRenderer } from "./DatabaseViewRenderer";
+import { DeferredRenderer } from "./DeferredRenderer";
+import { RenderDatabasePageTask } from "./RenderDatabasePageTask";
+import { DatabaseConfig } from "./SyncConfig";
+import { DatabaseTableRenderer } from "./DatabaseTableRenderer";
 
 export class ChildDatabaseRenderer {
   constructor(
     private readonly config: SyncConfig,
     private readonly publicApi: Client,
     private readonly deferredRenderer: DeferredRenderer,
-    private readonly tableRenderer: TableRenderer,
-    private readonly viewRenderer: DatabaseViewRenderer,
+    private readonly tableRenderer: DatabaseTableRenderer,
+    private readonly viewRenderer: DatabaseViewRenderer
   ) {}
 
-  async renderChildDatabase(databaseId: string): Promise<string> {
+  async renderChildDatabase(databaseId: string): Promise<Database> {
     const dbConfig = lookupDatabaseConfig(this.config, databaseId);
 
-    const msg = `<!-- included database ${databaseId} -->\n`;
-
     // no view was defined for this database, render as a plain inline table
     const allPages = await this.fetchPages(databaseId, dbConfig);
 
-    const isCmsDb = this.config.cmsDatabaseId !== databaseId;
-    if (isCmsDb && !dbConfig.views) {
-      return msg + await this.tableRenderer.renderTable(allPages, dbConfig);
+    const isCmsDb = this.config.cmsDatabaseId === databaseId;
+    const renderPages = isCmsDb || dbConfig.views;
+
+    if (renderPages) {
+      const entries = await this.queuePageRendering(allPages, dbConfig);
+      const markdown = await this.viewRenderer.renderViews(entries, dbConfig);
+
+      return {
+        config: dbConfig,
+        entries,
+        markdown,
+      };
     }
 
-    // queue all pages in the database for individual, deferred rendering
-    const prepareRenderPageTasks = allPages.map((x) =>
-      this.deferredRenderer.renderPage(x, dbConfig)
+    const entries = await this.queueEntryRendering(allPages, dbConfig);
+    const markdown = this.tableRenderer.renderTable(entries);
+
+    return {
+      config: dbConfig,
+      entries,
+      markdown,
+    };
+  }
+
+  private async queueEntryRendering(
+    allPages: Page[],
+    dbConfig: DatabaseConfig
+  ) {
+    const prepareRenderEntryTasks = allPages.map((x) =>
+      this.deferredRenderer.renderEntry(x, dbConfig)
     );
 
     // note: the await here is not actually starting to render the pages, however it prepares the page render task
-    const renderPageTasks = await Promise.all(prepareRenderPageTasks);
+    return await Promise.all(prepareRenderEntryTasks);
+  }
 
-    const db = {
-      pages: renderPageTasks,
-      config: dbConfig,
-    };
+  private async queuePageRendering(
+    allPages: Page[],
+    dbConfig: DatabaseConfig
+  ): Promise<RenderDatabasePageTask[]> {
+    const prepareRenderPageTasks = allPages.map((x) =>
+      this.deferredRenderer.renderPage(x, dbConfig)
+    );
 
-    return this.viewRenderer.renderViews(db);
+    // note: the await here is not actually starting to render the pages, however it prepares the page render task
+    return await Promise.all(prepareRenderPageTasks);
   }
 
   private async fetchPages(
     databaseId: string,
-    dbConfig: DatabaseConfig,
+    dbConfig: DatabaseConfig
   ): Promise<Page[]> {
     const db = await this.publicApi.databases.retrieve({
       database_id: databaseId,
@@ -62,7 +89,7 @@ export class ChildDatabaseRenderer {
 
     if (allPages.next_cursor) {
       throw new Error(
-        `Paging not implemented, db ${db.id} has more than 100 entries`,
+        `Paging not implemented, db ${db.id} has more than 100 entries`
       );
     }
 

src/Database.ts

@@ -1,7 +1,9 @@
 import { DatabaseConfig } from "./SyncConfig";
-import { RenderPageTask } from "./RenderPageTask";
+import { RenderDatabasePageTask } from "./RenderDatabasePageTask";
+import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask";
 
 export interface Database {
   config: DatabaseConfig;
-  pages: RenderPageTask[];
+  entries: (RenderDatabaseEntryTask | RenderDatabasePageTask)[];
+  markdown: string;
 }

src/DatabaseEntryRenderer.ts

@@ -0,0 +1,24 @@
+import { Page } from "@notionhq/client/build/src/api-types";
+
+import { PropertiesParser } from "./PropertiesParser";
+import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask";
+import { DatabaseConfig } from "./SyncConfig";
+
+export class DatabaseEntryRenderer {
+  constructor(private readonly propertiesParser: PropertiesParser) {}
+
+  async renderEntry(
+    page: Page,
+    config: DatabaseConfig
+  ): Promise<RenderDatabaseEntryTask> {
+    const props = await this.propertiesParser.parseProperties(page, config);
+
+    return {
+      id: page.id,
+      properties: {
+        keys: props.keys,
+        values: props.properties,
+      },
+    };
+  }
+}

src/PageRenderer.ts renamed to src/DatabasePageRenderer.ts

@@ -7,20 +7,20 @@ import { FrontmatterRenderer } from './FrontmatterRenderer';
 import { logger } from './logger';
 import { PropertiesParser } from './PropertiesParser';
 import { RecursiveBodyRenderer } from './RecursiveBodyRenderer';
-import { RenderPageTask as RenderPageTask } from './RenderPageTask';
+import { RenderDatabasePageTask as RenderDatabasePageTask } from "./RenderDatabasePageTask";
 import { slugify } from './slugify';
 import { DatabaseConfig } from './SyncConfig';
 
 const fs = fsc.promises;
 
-export class PageRenderer {
+export class DatabasePageRenderer {
   constructor(
     readonly propertiesParser: PropertiesParser,
     readonly frontmatterRenderer: FrontmatterRenderer,
     readonly bodyRenderer: RecursiveBodyRenderer
   ) {}
 
-  async renderPage(page: Page, config: DatabaseConfig): Promise<RenderPageTask> {
+  async renderPage(page: Page, config: DatabaseConfig): Promise<RenderDatabasePageTask> {
     if (page.archived){
       logger.warn(`rendering archived page ${page.url}`);
     }

src/DatabaseTableRenderer.ts

@@ -0,0 +1,34 @@
+import * as markdownTable from "./markdown-table";
+import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask";
+
+export class DatabaseTableRenderer {
+  public renderTable(entries: RenderDatabaseEntryTask[]): string {
+    const table: any[][] = [];
+
+    for (const page of entries) {
+      if (table.length === 0) {
+        const headers = Array.from(page.properties.keys.keys());
+        table[0] = headers;
+      }
+
+      const cols = Array.from(page.properties.keys.values()).map((c, i) =>
+        DatabaseTableRenderer.escapeTableCell(
+          page.properties.values[c]
+        )
+      );
+
+      table.push(cols);
+    }
+
+    return markdownTable.markdownTable(table);
+  }
+
+  static escapeTableCell(content: string | number | any): string {
+    // markdown table cells do not support newlines, however we can insert <br> elements instead
+    if (typeof content === "string") {
+      return content.replace(/\n/g, "<br>");
+    }
+
+    return content.toString();
+  }
+}