Skip to content

Commit ff4aea7

Browse files
initial commit
1 parent d96acf9 commit ff4aea7

26 files changed

+1342
-0
lines changed

.gitignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
node_modules
2+
.temp
3+
.cache
4+
.env
5+
yarn.lock
6+
dist/

README.md

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Notion Markdown CMS
2+
3+
Build markdown-based static sites with Notion.
4+
5+
1. Use Notion to write and organize pages
6+
2. `notion-markdown-cms sync` to build a markdown repository
7+
3. run your favourite static site generator (VuePress, Docusaurus, Gatsby, ...)
8+
9+
Success! 🚀
10+
11+
## Features
12+
13+
- uses the official Notion API only
14+
- written in typescript/javascript
15+
- renders page properties to frontmatter
16+
- recursively traverses the Notion Block graph to include database pages, child pages
17+
- renders an index file of all your pages so you can easily build Navs/Sidebars
18+
19+
### Supported Blocks
20+
21+
| Block Type | Supported | Notes |
22+
| ----------------- | ---------- | -------------------------------------------- |
23+
| Text | ✅ Yes | |
24+
| Heading | ✅ Yes | |
25+
| Image | ✅ Yes | |
26+
| Image Caption | ✅ Yes | |
27+
| Bulleted List | ✅ Yes | |
28+
| Numbered List | ✅ Yes | |
29+
| Quote | ✅ Yes | |
30+
| Callout | ✅ Yes | |
31+
| Column | ❌ Missing | |
32+
| iframe | ✅ Yes | |
33+
| Video | ❌ Missing | |
34+
| Divider | ✅ Yes | |
35+
| Link | ✅ Yes | |
36+
| Code | ✅ Yes | |
37+
| Web Bookmark | ✅ Yes | |
38+
| Toggle List | ✅ Yes | |
39+
| Page Links | ❌ Missing | |
40+
| Header | ✅ Yes | |
41+
| Databases | ✅ Yes | including child pages, inline tables planned |
42+
| Checkbox | ? | |
43+
| Table Of Contents | ? | |
44+
45+
### Configuration
46+
47+
## Related Projects and Inspiration
48+
49+
There are quite a few alternatives out there already, so why did we build `notion-markdown-cms`?
50+
Below table, albeit subjective, tries to answer this.
51+
52+
| Project | Notion API | Language | Rendering Engine |
53+
| ------------------------------------------------------------------------ | ------------- | ---------- | ------------------- |
54+
| [Nortion Markdown CMS](https://github.com/meshcloud/notion-markdown-cms) | ✅ official | TypeScript | Markdown + JS Index |
55+
| [Notion2GitHub](https://github.com/narkdown/notion2github) | ⚠️ unofficial | Python | Markdown |
56+
| [notion-cms](https://github.com/n6g7/notion-cms) | ⚠️ unofficial | TypeScript | React |
57+
| [vue-notion](https://github.com/janniks/vue-notion) | ⚠️ unofficial | JavaScript | Vue.js |
58+
| [react-notion](https://github.com/janniks/react-notion) | ⚠️ unofficial | JavaScript | React |

config.json.example

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"root": "8f1de8c578fb4590ad6fbb0dbe283338",
3+
"databases": {
4+
"fe9836a9-6557-4f17-8adb-a93d2584f35f": {
5+
"parentCategory": "mydb/",
6+
"sorts": [
7+
{
8+
"property": "Scope",
9+
"direction": "ascending"
10+
},
11+
{
12+
"property": "Cluster",
13+
"direction": "ascending"
14+
}
15+
],
16+
"properties": {
17+
"category": "scope",
18+
"include": [
19+
"Name",
20+
"Scope",
21+
"Cluster",
22+
"Summary"
23+
]
24+
}
25+
}
26+
}
27+
}

default.nix

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{ pkgs ? import <nixpkgs> { } }:
2+
3+
pkgs.mkShell {
4+
NIX_SHELL = "notioncms";
5+
6+
buildInputs = [
7+
pkgs.nodejs-14_x
8+
(pkgs.yarn.override {
9+
nodejs = pkgs.nodejs-14_x;
10+
})
11+
];
12+
}

package.json

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"name": "notion-markdown-cms",
3+
"version": "0.0.1",
4+
"main": "index.js",
5+
"repository": "git@github.com:meshcloud/cloudfoundation.git",
6+
"author": "Johannes Rudolph <jrudolph@meshcloud.io>",
7+
"license": "MIT",
8+
"scripts": {
9+
"sync": "ts-node src/index.ts",
10+
"build": "tsc",
11+
"prepare": "tsc"
12+
},
13+
"devDependencies": {
14+
"ts-node": "^10.2.1",
15+
"typescript": "^4.4.3",
16+
"@types/js-yaml": "^4.0.3",
17+
"@types/mime-types": "^2.1.1"
18+
},
19+
"dependencies": {
20+
"@notionhq/client": "^0.3.2",
21+
"chalk": "^4.1.2",
22+
"dotenv": "^10.0.0",
23+
"got": "^11.8.2",
24+
"js-yaml": "^4.1.0",
25+
"keyv-file": "^0.2.0",
26+
"mime-types": "^2.1.32",
27+
"slugify": "^1.6.0"
28+
}
29+
}

src/AssetWriter.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { promises as fs } from "fs";
2+
import got from "got";
3+
import { KeyvFile } from "keyv-file";
4+
import * as mime from "mime-types";
5+
6+
const cache = new KeyvFile({
7+
filename: ".cache/keyv.json",
8+
});
9+
10+
export class AssetWriter {
11+
constructor(readonly dir: string) {}
12+
13+
async store(name: string, buffer: Buffer) {
14+
await fs.writeFile(`${this.dir}/${name}`, buffer);
15+
}
16+
17+
async download(url: string, fileName: string) {
18+
// the got http lib promises to do proper user-agent compliant http caching
19+
// see https://github.com/sindresorhus/got/blob/main/documentation/cache.md
20+
21+
// unfortunately download caching does _not_ work with images hosted on notion
22+
// because the notion API does not return cache friendly signed S3 URLs, https://advancedweb.hu/cacheable-s3-signed-urls/
23+
const response = await got(url, { cache });
24+
25+
const ext = mime.extension(
26+
response.headers["content-type"] || "application/octet-stream"
27+
);
28+
const imageFile = fileName + "." + ext;
29+
30+
console.debug(
31+
`downloading (cached: ${response.isFromCache}): ${imageFile}`
32+
);
33+
await this.store(imageFile, response.rawBody);
34+
35+
return imageFile;
36+
}
37+
}

src/BlockRenderer.ts

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
import {
2+
Block as PublicBlock,
3+
BlockBase,
4+
Emoji,
5+
File,
6+
ExternalFile,
7+
ExternalFileWithCaption,
8+
FileWithCaption,
9+
ImageBlock,
10+
RichText,
11+
} from "@notionhq/client/build/src/api-types";
12+
13+
import * as markdownTable from "./markdown-table";
14+
import { AssetWriter } from "./AssetWriter";
15+
import { Database } from "./Database";
16+
import { DeferredRenderer } from "./DeferredRenderer";
17+
import { RichTextRenderer } from "./RichTextRenderer";
18+
import { logger } from "./logger";
19+
20+
const debug = require("debug")("blocks");
21+
22+
export interface CodeBlock extends BlockBase {
23+
type: "code";
24+
code: {
25+
text: RichText[];
26+
language: string;
27+
};
28+
}
29+
30+
export interface QuoteBlock extends BlockBase {
31+
type: "quote";
32+
code: {
33+
text: RichText[];
34+
language: string;
35+
};
36+
}
37+
38+
export interface CalloutBlock extends BlockBase {
39+
type: "callout";
40+
callout: {
41+
text: RichText[];
42+
icon: File | ExternalFile | Emoji;
43+
};
44+
}
45+
46+
export interface DividerBlock extends BlockBase {
47+
type: "divider";
48+
}
49+
50+
export interface ChildDatabaseBlock extends BlockBase {
51+
type: "child_database";
52+
}
53+
54+
// these are blocks that the notion API client code does not have proper typings for
55+
// for unknown reasons they removed types alltogether in v0.4 of the client
56+
// https://github.com/makenotion/notion-sdk-js/pulls?q=is%3Apr+is%3Aclosed#issuecomment-927781781
57+
export type Block =
58+
| PublicBlock
59+
| CodeBlock
60+
| QuoteBlock
61+
| CalloutBlock
62+
| DividerBlock
63+
| ChildDatabaseBlock;
64+
65+
export class BlockRenderer {
66+
constructor(
67+
private readonly deferredRenderer: DeferredRenderer,
68+
private readonly richText: RichTextRenderer
69+
) {}
70+
71+
async renderBlockLine(block: Block, assets: AssetWriter): Promise<string> {
72+
switch (block.type) {
73+
case "paragraph":
74+
return this.richText.renderMarkdown(block.paragraph.text);
75+
case "heading_1":
76+
return "# " + this.richText.renderMarkdown(block.heading_1.text);
77+
case "heading_2":
78+
return "## " + this.richText.renderMarkdown(block.heading_2.text);
79+
case "heading_3":
80+
return "### " + this.richText.renderMarkdown(block.heading_3.text);
81+
case "bulleted_list_item":
82+
return (
83+
"- " + this.richText.renderMarkdown(block.bulleted_list_item.text)
84+
);
85+
case "numbered_list_item":
86+
return (
87+
"1. " + this.richText.renderMarkdown(block.numbered_list_item.text)
88+
);
89+
case "to_do":
90+
return "[ ] " + this.richText.renderMarkdown(block.to_do.text);
91+
case "image":
92+
return await this.renderImage(block, assets);
93+
case "quote":
94+
block as any;
95+
return "> " + this.richText.renderMarkdown((block as any).quote.text);
96+
case "code":
97+
return (
98+
"```" +
99+
block.code.language +
100+
"\n" +
101+
this.richText.renderMarkdown(block.code.text) +
102+
"\n```"
103+
);
104+
case "callout":
105+
return (
106+
"> " +
107+
this.renderIcon(block.callout.icon) +
108+
" " +
109+
this.richText.renderMarkdown(block.callout.text)
110+
);
111+
case "divider":
112+
return "---";
113+
case "child_database":
114+
// queue all pages in the database for individual, deferred rendering
115+
const db = await this.deferredRenderer.renderDatabasePages(block.id);
116+
const msg = `<!-- included database ${block.id} -->\n`;
117+
118+
// todo: make this nicer, e.g. render multi tables
119+
return msg + this.renderTables(db);
120+
121+
case "toggle":
122+
case "child_page":
123+
case "embed":
124+
case "bookmark":
125+
case "video":
126+
case "file":
127+
case "pdf":
128+
case "audio":
129+
case "unsupported":
130+
default:
131+
return this.renderUnsupported(
132+
`unsupported block type: ${block.type}`,
133+
block
134+
);
135+
}
136+
}
137+
138+
renderTables(db: Database) {
139+
// todo: handle empty page
140+
const props = db.pages[0].properties;
141+
142+
const table: any[][] = [];
143+
144+
const headers = Array.from(props.keys.keys());
145+
table[0] = headers;
146+
147+
const cols = Array.from(props.keys.values());
148+
db.pages.forEach((r) =>
149+
table.push(
150+
cols.map((c, i) => {
151+
const content = escapeTableCell(r.properties.values[c]);
152+
return i == 0
153+
? this.renderMarkdownLink(content, r.file.substr("docs".length)) // make the first cell a relative link to the page
154+
: content;
155+
})
156+
)
157+
);
158+
159+
return markdownTable.markdownTable(table);
160+
}
161+
162+
private renderIcon(icon: File | ExternalFile | Emoji): string {
163+
switch (icon.type) {
164+
case "emoji":
165+
return icon.emoji;
166+
case "file":
167+
case "external":
168+
return this.renderUnsupported(
169+
`unsupported icon type: ${icon.type}`,
170+
icon
171+
);
172+
}
173+
}
174+
175+
async renderImage(block: ImageBlock, assets: AssetWriter): Promise<string> {
176+
const url = this.parseUrl(block.image);
177+
178+
const imageFile = await assets.download(url, block.id);
179+
180+
// todo: caption support
181+
const markdown = `![image-${block.id}](./${imageFile})`;
182+
return markdown;
183+
}
184+
185+
private parseUrl(image: FileWithCaption | ExternalFileWithCaption) {
186+
switch (image.type) {
187+
case "external":
188+
return image.external.url;
189+
case "file":
190+
return image.file.url;
191+
}
192+
}
193+
194+
private renderMarkdownLink(text: string, url: string): string {
195+
return `[${text}](${url})`;
196+
}
197+
198+
private renderUnsupported(msg: string, obj: any): string {
199+
logger.warn(msg);
200+
debug(msg + "\n%O", obj);
201+
202+
return `<!-- ${msg} -->`;
203+
}
204+
}
205+
206+
function escapeTableCell(content: string): string {
207+
// markdown table cells do not support newlines, however we can insert <br> elements instead
208+
return content.replace(/\n/g, "<br>");
209+
}

src/Database.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { DatabaseConfig } from "./SyncConfig";
2+
import { RenderPageTask } from "./RenderedPageTask";
3+
4+
export interface Database {
5+
config: DatabaseConfig;
6+
pages: RenderPageTask[];
7+
}

0 commit comments

Comments
 (0)