fix: correctly handle configured outDirs, indexPath

Author: JohannesRudolph

jest.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+  transform: {
+    "^.+\\.ts?$": "ts-jest",
+  },
+  testEnvironment: "node",
+  testRegex: "./src/.*\\.(test|spec)?\\.(ts|ts)$",
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"],
+  roots: ["<rootDir>/src"],
+};

package.json

@@ -13,13 +13,17 @@
   ],
   "scripts": {
     "build": "tsc",
-    "prepare": "tsc"
+    "prepare": "tsc",
+    "test": "jest"
   },
   "devDependencies": {
-    "ts-node": "^10.2.1",
-    "typescript": "^4.4.3",
+    "@types/jest": "^27.0.2",
     "@types/js-yaml": "^4.0.3",
-    "@types/mime-types": "^2.1.1"
+    "@types/mime-types": "^2.1.1",
+    "jest": "^27.3.1",
+    "ts-jest": "^27.0.7",
+    "ts-node": "^10.2.1",
+    "typescript": "^4.4.3"
   },
   "dependencies": {
     "@notionhq/client": "^0.3.2",

src/DatabasePageRenderer.ts

@@ -2,22 +2,27 @@ import { Client } from "@notionhq/client";
 import { DatabaseConfig } from "./SyncConfig";
 import { Database } from "./Database";
 import { DeferredRenderer } from "./DeferredRenderer";
+import { SyncConfig } from ".";
 
 export class DatabasePageRenderer {
   constructor(
     readonly publicApi: Client,
     readonly deferredRenderer: DeferredRenderer,
-    readonly config: Record<string, DatabaseConfig>
+    readonly config: SyncConfig
   ) {}
 
   async renderDatabase(databaseId: string): Promise<Database> {
-    const dbConfig: DatabaseConfig = this.config[databaseId] || {
-      outSubDir: "",
-      pageCategoryValuePrefix: "",
+    const fallbackDbConfig: DatabaseConfig = {
+      outDir:
+        databaseId === this.config.cmsDatabaseId
+          ? this.config.outDir
+          : this.config.outDir + "/" + databaseId,
       properties: {
-        category: "category",
+        category: "Category",
       },
     };
+    const dbConfig: DatabaseConfig =
+      this.config.databases[databaseId] || fallbackDbConfig;
 
     const db = await this.publicApi.databases.retrieve({
       database_id: databaseId,

src/PageRenderer.ts

@@ -31,15 +31,18 @@ export class PageRenderer {
       this.throwMissingRequiredProperty("name", page);
     }
 
-    const category = props.values[props.keys.get(config.properties.category)!!];
-    if (category) {
-      this.throwMissingRequiredProperty(config.properties.category, page);
+    const category = props.values["category"];
+    if (!category) {
+      this.throwMissingRequiredProperty("category", page);
     }
 
+    debug(props);
+
     const nameSlug = slugify(name);
-    const categorySlug = config.pageCategoryValuePrefix + slugify(category);
+    const categorySlug = slugify(category);
 
-    const file = `${config.outDir}/${nameSlug}.md`;
+    const destDir = `${config.outDir}/${categorySlug}`;
+    const file = `${destDir}/${nameSlug}.md`;
 
     // Design: all the rendering performance could be greatly enhanced writing directly to output streams instead
     // of concatenating all in memory. OTOH naively concatenatic strings is straightforward, easier to debug and rendering
@@ -51,7 +54,7 @@ export class PageRenderer {
       file,
       properties: props,
       render: async () => {
-        const assetWriter = new AssetWriter(config.outDir);
+        const assetWriter = new AssetWriter(destDir);
 
         const frontmatter = this.frontmatterRenderer.renderFrontmatter(
           props.values
@@ -62,7 +65,7 @@ export class PageRenderer {
           assetWriter
         );
 
-        await fs.mkdir(config.outDir, { recursive: true });
+        await fs.mkdir(destDir, { recursive: true });
         await fs.writeFile(file, frontmatter + body);
 
         logger.info("wrote: " + file);

src/PropertiesParser.spec.ts

@@ -0,0 +1,75 @@
+import { PropertiesParser } from "./PropertiesParser";
+import { DatabaseConfig } from "./SyncConfig";
+
+describe("PropertiesParser", () => {
+  describe("filter", () => {
+    test("preserves all properties and adds conventional with no include filter", async () => {
+      const sut = new PropertiesParser({} as any);
+
+      const props = {
+        values: { order: 30, category: "Tools", name: "Terraform" },
+        keys: new Map([
+          ["order", "order"],
+          ["Category", "category"],
+          ["Name", "name"],
+        ]),
+      };
+
+      const config: DatabaseConfig = {
+        outDir: "db/",
+        properties: {
+          category: "Category",
+        },
+      };
+
+      const result = sut.filter(config, props);
+      const expected = {
+        keys: props.keys,
+        values: {
+          order: 30,
+          category: "Tools",
+          name: "Terraform",
+          title: "Terraform",
+        },
+      };
+
+      expect(result).toEqual(expected);
+    });
+
+    test("filters according to include filter", async () => {
+      const sut = new PropertiesParser({} as any);
+
+      const props = {
+        values: { order: 30, category: "Tools", name: "Terraform" },
+        keys: new Map([
+          ["order", "order"],
+          ["Category", "category"],
+          ["Name", "name"],
+        ]),
+      };
+
+      const config: DatabaseConfig = {
+        outDir: "db/",
+        properties: {
+          category: "Category",
+          include: ["Name", "Category"],
+        },
+      };
+
+      const result = sut.filter(config, props);
+      const expected = {
+        keys: new Map([
+          ["Name", "name"],
+          ["Category", "category"],
+        ]),
+        values: {
+          category: "Tools",
+          name: "Terraform",
+          title: "Terraform",
+        },
+      };
+
+      expect(result).toEqual(expected);
+    });
+  });
+});

src/PropertiesParser.ts

@@ -75,7 +75,8 @@ export class PropertiesParser {
     // patch properties by convention
     // vuepress expects a title and category property
     properties["title"] = input.values["name"]; // notion API always calls it name
-    properties["category"] = input.values[config.properties.category];
+    properties["category"] =
+      input.values[input.keys.get(config.properties.category)!!];
 
     // vuepress relies on files being called README.md to make "category home" pages
     // however the name "README" does not read nice in a sidebar menu, so we label these pages "Introduction"

src/SyncConfig.ts

@@ -4,6 +4,12 @@ export interface SyncConfig {
   /**
    * Id of the Notion database containing the CMS content.
    * Traversing the block graph starts here.
+   * 
+   * By convention this database must have the following properties
+   * 
+   * - Name: page name
+   * - Category: page category
+   * - sort: number to control sorting of pages in sidebar
    */
   cmsDatabaseId: string;
 
@@ -31,18 +37,10 @@ export interface DatabaseConfig {
   /**
    * The output directory where the sync will place pages of this database.
    * 
-   * Example: "docs/mydb/"
+   * Example: docs/mydb"
    */
   outDir: string;
 
-  /**
-   * The prefix to apply to the category value of all pages.
-   * This is useful to create a unique category name for all pages of this database.
-   * 
-   * Example: "mydb/"
-   */
-  pageCategoryValuePrefix: string;
-
   /**
    * Notion API https://developers.notion.com/reference/post-database-query#post-database-query-sort
    */

src/slugify.spec.ts

@@ -0,0 +1,11 @@
+import { slugify } from "./slugify";
+
+describe("slugify", () => {
+  test("does not preserve forward slashes", () => {
+    expect(slugify("CFMM / core")).toEqual("cfmm-core");
+  });
+
+  test("does not emojis", () => {
+    expect(slugify("cfmm/🏢 Core")).toEqual("cfmm-core");
+  });
+});

src/sync.ts

@@ -26,10 +26,11 @@ export async function sync(notionApiToken: string, config: SyncConfig) {
     frontmatterRenderer,
     bodyRenderer
   );
+
   const dbRenderer = new DatabasePageRenderer(
     publicApi,
     deferredRenderer,
-    config.databases
+    config
   );
 
   deferredRenderer.initialize(dbRenderer, pageRenderer);
@@ -39,7 +40,7 @@ export async function sync(notionApiToken: string, config: SyncConfig) {
   await deferredRenderer.process();
 
   await fs.writeFile(
-    "docs/.vuepress/index.ts",
+    config.indexPath,
     `export const index = ${JSON.stringify(
       deferredRenderer.getRenderedPages(),
       null,

tsconfig.json

@@ -1,6 +1,8 @@
 {
   "compilerOptions": {
-    "lib": ["ES2020"],
+    "lib": [
+      "ES2020"
+    ],
     "target": "ES2020",
     "module": "CommonJS",
     "allowJs": true,
@@ -19,7 +21,18 @@
     "noUnusedParameters": false,
     "noUnusedLocals": true,
     "outDir": "./dist",
-    "typeRoots": ["node_modules/@types"]
+    "typeRoots": [
+      "node_modules/@types"
+    ],
+    "types": [
+      "node",
+      "@types/jest"
+    ]
   },
-  "include": ["src/**.ts"]
-}
+  "include": [
+    "src/**.ts"
+  ],
+  "exclude": [
+    "src/**/*.spec.ts"
+  ]
+}