feat(ai): Add .md extension to provide pages in markdown for LLMs (#13994)

Adds support for `.md` at the end of every pre-rendered path. Does this
by a hack where we rewrite those paths to `public/md-exports/...`. The
contents of this directory are generated _after_ `next build` by
scraping all html files under `.next/server/app`

Not ideal but looks like the easiest path for now.

---------

Co-authored-by: Burak Yigit Kaya <byk@sentry.io>

Author: 2 people

.gitignore

@@ -4,6 +4,9 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
+# Ignore generated export markdown files 
+/public/md-exports/
+
 # Runtime data
 pids
 *.pid

next.config.ts

@@ -1,7 +1,7 @@
 import {codecovNextJSWebpackPlugin} from '@codecov/nextjs-webpack-plugin';
 import {withSentryConfig} from '@sentry/nextjs';
 
-import {redirects} from './redirects';
+import {redirects} from './redirects.js';
 
 const outputFileTracingExcludes = process.env.NEXT_PUBLIC_DEVELOPER_DOCS
   ? {
@@ -55,6 +55,12 @@ const nextConfig = {
     DEVELOPER_DOCS_: process.env.NEXT_PUBLIC_DEVELOPER_DOCS,
   },
   redirects,
+  rewrites: async () => [
+    {
+      source: '/:path*.md',
+      destination: '/md-exports/:path*.md',
+    },
+  ],
   sassOptions: {
     silenceDeprecations: ['legacy-js-api'],
   },

package.json

@@ -18,8 +18,10 @@
     "dev": "yarn enforce-redirects && concurrently \"yarn sidecar\" \"node ./src/hotReloadWatcher.mjs\" \"next dev\"",
     "dev:developer-docs": "yarn enforce-redirects && NEXT_PUBLIC_DEVELOPER_DOCS=1 yarn dev",
     "build:developer-docs": "yarn enforce-redirects && git submodule init && git submodule update && NEXT_PUBLIC_DEVELOPER_DOCS=1 yarn build",
-    "build": "yarn enforce-redirects && next build",
+    "build": "yarn enforce-redirects && next build && yarn generate-md-exports",
+    "generate-md-exports": "node scripts/generate-md-exports.mjs",
     "vercel:build:developer-docs": "yarn enforce-redirects && git submodule init && git submodule update && NEXT_PUBLIC_DEVELOPER_DOCS=1 yarn build",
+    "start:dev": "NODE_ENV=development yarn build && yarn start",
     "start": "next start",
     "lint": "next lint",
     "lint:ts": "tsc --skipLibCheck",
@@ -61,6 +63,7 @@
     "framer-motion": "^10.12.16",
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
+    "hast-util-select": "^6.0.4",
     "hast-util-to-jsx-runtime": "^2.3.2",
     "hastscript": "^8.0.0",
     "image-size": "^1.2.1",
@@ -85,21 +88,26 @@
     "react-select": "^5.7.3",
     "react-textarea-autosize": "^8.5.3",
     "rehype-autolink-headings": "^7.1.0",
+    "rehype-parse": "^9.0.1",
     "rehype-preset-minify": "^7.0.0",
     "rehype-prism-diff": "^1.1.2",
     "rehype-prism-plus": "^1.6.3",
+    "rehype-remark": "^10.0.1",
     "rehype-stringify": "^10.0.0",
-    "remark-gfm": "^4.0.0",
+    "remark-gfm": "^4.0.1",
     "remark-mdx-images": "^3.0.0",
     "remark-parse": "^11.0.0",
     "remark-prism": "^1.3.6",
+    "remark-stringify": "^11.0.0",
     "rss": "^1.2.2",
     "sass": "^1.69.5",
     "search-insights": "^2.17.2",
     "server-only": "^0.0.1",
     "sharp": "^0.33.4",
     "tailwindcss-scoped-preflight": "^3.0.4",
-    "textarea-markdown-editor": "^1.0.4"
+    "textarea-markdown-editor": "^1.0.4",
+    "unified": "^11.0.5",
+    "unist-util-remove": "^4.0.0"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.15.0",

scripts/generate-md-exports.mjs

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+
+import {fileURLToPath} from 'url';
+
+import {selectAll} from 'hast-util-select';
+import {existsSync} from 'node:fs';
+import {mkdir, opendir, readFile, rm, writeFile} from 'node:fs/promises';
+import {cpus} from 'node:os';
+import * as path from 'node:path';
+import {isMainThread, parentPort, Worker, workerData} from 'node:worker_threads';
+import rehypeParse from 'rehype-parse';
+import rehypeRemark from 'rehype-remark';
+import remarkGfm from 'remark-gfm';
+import remarkStringify from 'remark-stringify';
+import {unified} from 'unified';
+import {remove} from 'unist-util-remove';
+
+async function createWork() {
+  let root = process.cwd();
+  while (!existsSync(path.join(root, 'package.json'))) {
+    const parent = path.dirname(root);
+    if (parent === root) {
+      throw new Error('Could not find package.json in parent directories');
+    }
+    root = parent;
+  }
+  const INPUT_DIR = path.join(root, '.next', 'server', 'app');
+  const OUTPUT_DIR = path.join(root, 'public', 'md-exports');
+
+  console.log(`🚀 Starting markdown generation from: ${INPUT_DIR}`);
+  console.log(`📁 Output directory: ${OUTPUT_DIR}`);
+
+  // Clear output directory
+  await rm(OUTPUT_DIR, {recursive: true, force: true});
+  await mkdir(OUTPUT_DIR, {recursive: true});
+
+  // On a 16-core machine, 8 workers were optimal (and slightly faster than 16)
+  const numWorkers = Math.max(Math.floor(cpus().length / 2), 2);
+  const workerTasks = new Array(numWorkers).fill(null).map(() => []);
+
+  console.log(`🔎 Discovering files to convert...`);
+
+  let numFiles = 0;
+  let workerIdx = 0;
+  // Need a high buffer size here otherwise Node skips some subdirectories!
+  // See https://github.com/nodejs/node/issues/48820
+  const dir = await opendir(INPUT_DIR, {recursive: true, bufferSize: 1024});
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith('.html') && dirent.isFile()) {
+      const sourcePath = path.join(dirent.parentPath || dirent.path, dirent.name);
+      const targetDir = path.join(
+        OUTPUT_DIR,
+        path.relative(INPUT_DIR, dirent.parentPath || dirent.path)
+      );
+      await mkdir(targetDir, {recursive: true});
+      const targetPath = path.join(targetDir, dirent.name.slice(0, -5) + '.md');
+      workerTasks[workerIdx].push({sourcePath, targetPath});
+      workerIdx = (workerIdx + 1) % numWorkers;
+      numFiles++;
+    }
+  }
+
+  console.log(`📄 Converting ${numFiles} files with ${numWorkers} workers...`);
+
+  const selfPath = fileURLToPath(import.meta.url);
+  const workerPromises = new Array(numWorkers - 1).fill(null).map((_, idx) => {
+    return new Promise((resolve, reject) => {
+      const worker = new Worker(selfPath, {workerData: workerTasks[idx]});
+      let hasErrors = false;
+      worker.on('message', data => {
+        if (data.failedTasks.length === 0) {
+          console.log(`✅ Worker[${idx}]: ${data.success} files successfully.`);
+        } else {
+          hasErrors = true;
+          console.error(`❌ Worker[${idx}]: ${data.failedTasks.length} files failed:`);
+          console.error(data.failedTasks);
+        }
+      });
+      worker.on('error', reject);
+      worker.on('exit', code => {
+        if (code !== 0) {
+          reject(new Error(`Worker[${idx}] stopped with exit code ${code}`));
+        } else {
+          hasErrors ? reject(new Error(`Worker[${idx}] had some errors.`)) : resolve();
+        }
+      });
+    });
+  });
+  // The main thread can also process tasks -- That's 65% more bullet per bullet! -Cave Johnson
+  workerPromises.push(processTaskList(workerTasks[workerTasks.length - 1]));
+
+  await Promise.all(workerPromises);
+
+  console.log(`📄 Generated ${numFiles} markdown files from HTML.`);
+  console.log('✅ Markdown export generation complete!');
+}
+
+async function genMDFromHTML(source, target) {
+  const text = await readFile(source, {encoding: 'utf8'});
+  await writeFile(
+    target,
+    String(
+      await unified()
+        .use(rehypeParse)
+        // Need the `main div > hgroup` selector for the headers
+        .use(() => tree => selectAll('main div > hgroup, div#main', tree))
+        // If we don't do this wrapping, rehypeRemark just returns an empty string -- yeah WTF?
+        .use(() => tree => ({
+          type: 'element',
+          tagName: 'div',
+          properties: {},
+          children: tree,
+        }))
+        .use(rehypeRemark, {
+          document: false,
+          handlers: {
+            // Remove buttons as they usually get confusing in markdown, especially since we use them as tab headers
+            button() {},
+          },
+        })
+        // We end up with empty inline code blocks, probably from some tab logic in the HTML, remove them
+        .use(() => tree => remove(tree, {type: 'inlineCode', value: ''}))
+        .use(remarkGfm)
+        .use(remarkStringify)
+        .process(text)
+    )
+  );
+}
+
+async function processTaskList(tasks) {
+  const failedTasks = [];
+  for (const {sourcePath, targetPath} of tasks) {
+    try {
+      await genMDFromHTML(sourcePath, targetPath);
+    } catch (error) {
+      failedTasks.push({sourcePath, targetPath, error});
+    }
+  }
+  return {success: tasks.length - failedTasks.length, failedTasks};
+}
+
+async function doWork(tasks) {
+  parentPort.postMessage(await processTaskList(tasks));
+}
+
+if (isMainThread) {
+  await createWork();
+} else {
+  await doWork(workerData);
+}

src/components/apiExamples/apiExamples.tsx

@@ -129,7 +129,11 @@ export function ApiExamples({api}: Props) {
           </button>
         </div>
         <pre className={`${styles['api-block-example']} relative`}>
-          <div className={codeBlockStyles.copied} style={{opacity: showCopied ? 1 : 0}}>
+          <div
+            data-mdast="ignore"
+            className={codeBlockStyles.copied}
+            style={{opacity: showCopied ? 1 : 0}}
+          >
             Copied
           </div>
           {selectedTabView === 0 &&

src/components/breadcrumbs/index.tsx

@@ -24,7 +24,7 @@ export function Breadcrumbs({leafNode}: BreadcrumbsProps) {
   }
 
   return (
-    <ul className="list-none flex p-0 flex-wrap" style={{margin: 0}}>
+    <ul className="list-none flex p-0 flex-wrap float-left" style={{margin: 0}}>
       {breadcrumbs.map(b => {
         return (
           <li className={styles['breadcrumb-item']} key={b.to}>

src/components/codeBlock/index.tsx

@@ -55,7 +55,11 @@ export function CodeBlock({filename, language, children}: CodeBlockProps) {
           </button>
         )}
       </div>
-      <div className={styles.copied} style={{opacity: showCopied ? 1 : 0}}>
+      <div
+        data-mdast="ignore"
+        className={styles.copied}
+        style={{opacity: showCopied ? 1 : 0}}
+      >
         Copied
       </div>
       <div ref={codeRef}>

src/components/docPage/index.tsx

@@ -1,6 +1,8 @@
 import {ReactNode} from 'react';
+import Link from 'next/link';
 
 import {getCurrentGuide, getCurrentPlatform, nodeForPath} from 'sentry-docs/docTree';
+import Markdown from 'sentry-docs/icons/Markdown';
 import {serverContext} from 'sentry-docs/serverContext';
 import {FrontMatter} from 'sentry-docs/types';
 import {PaginationNavNode} from 'sentry-docs/types/paginationNavNode';
@@ -81,7 +83,17 @@ export function DocPage({
             <div className="mb-4">
               <Banner />
             </div>
-            {leafNode && <Breadcrumbs leafNode={leafNode} />}
+            <div className="overflow-hidden">
+              {leafNode && <Breadcrumbs leafNode={leafNode} />}{' '}
+              <Link
+                rel="nofollow"
+                className="float-right"
+                href={`/${pathname}.md`}
+                title="Markdown version of this page"
+              >
+                <Markdown className="flex p-0 flex-wrap" width={24} height={24} />
+              </Link>
+            </div>
             <div>
               <hgroup>
                 <h1>{frontMatter.title}</h1>

src/icons/Markdown.tsx

@@ -0,0 +1,15 @@
+function Markdown({width = 16, height = 16, ...props}: React.SVGAttributes<SVGElement>) {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      width={width}
+      height={height}
+      viewBox="0 0 32 32"
+      fill="currentColor"
+      {...props}
+    >
+      <path d="M 2.875 6 C 1.320313 6 0 7.253906 0 8.8125 L 0 23.1875 C 0 24.746094 1.320313 26 2.875 26 L 29.125 26 C 30.679688 26 32 24.746094 32 23.1875 L 32 8.8125 C 32 7.253906 30.679688 6 29.125 6 Z M 2.875 8 L 29.125 8 C 29.640625 8 30 8.382813 30 8.8125 L 30 23.1875 C 30 23.617188 29.640625 24 29.125 24 L 2.875 24 C 2.359375 24 2 23.617188 2 23.1875 L 2 8.8125 C 2 8.382813 2.359375 8 2.875 8 Z M 5 11 L 5 21 L 8 21 L 8 14.34375 L 11 18.3125 L 14 14.34375 L 14 21 L 17 21 L 17 11 L 14 11 L 11 15 L 8 11 Z M 22 11 L 22 16 L 19 16 L 23.5 21 L 28 16 L 25 16 L 25 11 Z" />
+    </svg>
+  );
+}
+export default Markdown;

src/mdx.ts

@@ -429,9 +429,9 @@ export async function getFileBySlug(slug: string) {
             ],
           },
         ],
-        [rehypePrismPlus, {ignoreMissing: true}],
+        [rehypePrismPlus, {ignoreMissing: true}] as any,
         rehypeOnboardingLines,
-        [rehypePrismDiff, {remove: true}],
+        [rehypePrismDiff, {remove: true}] as any,
         rehypePresetMinify,
       ];
       return options;