feat(md): Rewrite URLs to be absolute and to .md versions (#14196)

Author: BYK

package.json

@@ -45,6 +45,7 @@
     "@google-cloud/storage": "^7.7.0",
     "@mdx-js/loader": "^3.0.0",
     "@mdx-js/react": "^3.0.0",
+    "@pondorasti/remark-img-links": "^1.0.8",
     "@popperjs/core": "^2.11.8",
     "@prettier/plugin-xml": "^3.3.1",
     "@radix-ui/colors": "^3.0.0",
@@ -97,6 +98,7 @@
     "rehype-remark": "^10.0.1",
     "rehype-stringify": "^10.0.0",
     "remark-gfm": "^4.0.1",
+    "remark-link-rewrite": "^1.0.7",
     "remark-mdx-images": "^3.0.0",
     "remark-parse": "^11.0.0",
     "remark-prism": "^1.3.6",

scripts/generate-md-exports.mjs

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 /* eslint-disable no-console */
 import {ListObjectsV2Command, PutObjectCommand, S3Client} from '@aws-sdk/client-s3';
+import imgLinks from '@pondorasti/remark-img-links';
 import {selectAll} from 'hast-util-select';
 import {createHash} from 'node:crypto';
 import {createReadStream, createWriteStream, existsSync} from 'node:fs';
@@ -20,11 +21,13 @@ import {
 import rehypeParse from 'rehype-parse';
 import rehypeRemark from 'rehype-remark';
 import remarkGfm from 'remark-gfm';
+import RemarkLinkRewrite from 'remark-link-rewrite';
 import remarkStringify from 'remark-stringify';
 import {unified} from 'unified';
 import {remove} from 'unist-util-remove';
 
-const CACHE_VERSION = 2;
+const DOCS_BASE_URL = 'https://docs.sentry.io/';
+const CACHE_VERSION = 3;
 const CACHE_COMPRESS_LEVEL = 4;
 const R2_BUCKET = process.env.NEXT_PUBLIC_DEVELOPER_DOCS
   ? 'sentry-develop-docs'
@@ -247,6 +250,19 @@ async function genMDFromHTML(source, target, {cacheDir, noCache}) {
           }),
         },
       })
+      .use(RemarkLinkRewrite, {
+        // There's a chance we might be changing absolute URLs here
+        // We'll check the code base and fix that later
+        replacer: url => {
+          const mdUrl = new URL(url, DOCS_BASE_URL);
+          const newPathName = mdUrl.pathname.replace(/\/?$/, '');
+          if (path.extname(newPathName) === '') {
+            mdUrl.pathname = `${newPathName}.md`;
+          }
+          return mdUrl;
+        },
+      })
+      .use(imgLinks, {absolutePath: DOCS_BASE_URL})
       // We end up with empty inline code blocks, probably from some tab logic in the HTML, remove them
       .use(() => tree => remove(tree, {type: 'inlineCode', value: ''}))
       .use(remarkGfm)

yarn.lock

@@ -2663,6 +2663,13 @@
   resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz"
   integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
 
+"@pondorasti/remark-img-links@^1.0.8":
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/@pondorasti/remark-img-links/-/remark-img-links-1.0.8.tgz#1ba8246834c0a5b6b13a9b25a77827d49f6bea25"
+  integrity sha512-UpW5AfimYi7EdkL9qvQ4trzIzgi5KERdd4KHN8fkVWrh2BqCzKuheJbAzSgIxJr0opQ7NvaIwf+JRjV/fBRdrA==
+  dependencies:
+    unist-util-visit "1.4.0"
+
 "@popperjs/core@^2.11.8":
   version "2.11.8"
   resolved "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz"
@@ -12174,6 +12181,13 @@ remark-gfm@^4.0.1:
     remark-stringify "^11.0.0"
     unified "^11.0.0"
 
+remark-link-rewrite@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/remark-link-rewrite/-/remark-link-rewrite-1.0.7.tgz#c8b5ad3708b0a8f6dd93caeab4c4d4c3e7e0d3a4"
+  integrity sha512-jfmXL8VL3O7Ca4g+Ya0A0S9LIpVhCUSqMHiei4N8Z8Mx9z/oEI1HDoa1ZcF9/tS0I7NwydGsPkw2zT4eeoD6+Q==
+  dependencies:
+    unist-util-visit "^4.1.1"
+
 remark-mdx-frontmatter@^4.0.0:
   version "4.0.0"
   resolved "https://registry.npmjs.org/remark-mdx-frontmatter/-/remark-mdx-frontmatter-4.0.0.tgz"
@@ -13423,6 +13437,11 @@ unist-util-generated@^2.0.0:
   resolved "https://registry.npmjs.org/unist-util-generated/-/unist-util-generated-2.0.1.tgz"
   integrity sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A==
 
+unist-util-is@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-3.0.0.tgz#d9e84381c2468e82629e4a5be9d7d05a2dd324cd"
+  integrity sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A==
+
 unist-util-is@^5.0.0:
   version "5.2.1"
   resolved "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.2.1.tgz"
@@ -13504,6 +13523,13 @@ unist-util-stringify-position@^4.0.0:
   dependencies:
     "@types/unist" "^3.0.0"
 
+unist-util-visit-parents@^2.0.0:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz#25e43e55312166f3348cae6743588781d112c1e9"
+  integrity sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==
+  dependencies:
+    unist-util-is "^3.0.0"
+
 unist-util-visit-parents@^5.0.0, unist-util-visit-parents@^5.1.1:
   version "5.1.3"
   resolved "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-5.1.3.tgz"
@@ -13520,7 +13546,14 @@ unist-util-visit-parents@^6.0.0:
     "@types/unist" "^3.0.0"
     unist-util-is "^6.0.0"
 
-unist-util-visit@^4.0.0, unist-util-visit@^4.1.0:
+unist-util-visit@1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-1.4.0.tgz#1cb763647186dc26f5e1df5db6bd1e48b3cc2fb1"
+  integrity sha512-FiGu34ziNsZA3ZUteZxSFaczIjGmksfSgdKqBfOejrrfzyUy5b7YrlzT1Bcvi+djkYDituJDy2XB7tGTeBieKw==
+  dependencies:
+    unist-util-visit-parents "^2.0.0"
+
+unist-util-visit@^4.0.0, unist-util-visit@^4.1.0, unist-util-visit@^4.1.1:
   version "4.1.2"
   resolved "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-4.1.2.tgz"
   integrity sha512-MSd8OUGISqHdVvfY9TPhyK2VdUrPgxkUtWSuMHF6XAAFuL4LokseigBnZtPnJMu+FbynTkFNnFlyjxpVKujMRg==