fix some broken images in remote mdx, use ast replacing instead regex (#434)

Author: Dimitri POSTOLOV

website/pages/en/firehose/[[...slug]].mdx

@@ -2,6 +2,7 @@ import { RemoteContent } from 'nextra/data'
 import { buildDynamicMDX } from 'nextra/remote'
 import { getPageMap } from '@/src/getPageMap'
 import { remarkReplaceLinks } from '@/src/remarkReplaceLinks'
+import { remarkReplaceImages } from '@/src/remarkReplaceImages'
 import { replaceGitBookContent } from '@/src/replaceGitBookContent'
 import json from '@/remote-files/firehose.json'
 
@@ -21,21 +22,15 @@ export async function getStaticProps({ params }) {
   const baseURL = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/${docsPath}${foundPath}`
   const response = await fetch(baseURL)
   const content = await response.text()
-  const mdx = await buildDynamicMDX(
-    replaceGitBookContent({
-      content,
-      user,
-      repo,
-      branch,
-      docsPath,
-    }),
-    {
-      mdxOptions: {
-        remarkPlugins: [[remarkReplaceLinks, { foundPath, basePath: '/firehose/' }]],
-      },
-      codeHighlight: false,
-    }
-  )
+  const mdx = await buildDynamicMDX(replaceGitBookContent({ content }), {
+    mdxOptions: {
+      remarkPlugins: [
+        [remarkReplaceLinks, { foundPath, basePath: '/firehose/' }],
+        [remarkReplaceImages, { assetsBasePath: `${user}/${repo}/${branch}/${docsPath}` }],
+      ],
+    },
+    codeHighlight: false,
+  })
   const parsedData = JSON.parse(mdx.__nextra_dynamic_opts)
   // remove title from frontMatter if it's the same as the title of the page
   if (parsedData.title === parsedData.frontMatter.title) {

website/pages/en/substreams/[[...slug]].mdx

@@ -2,6 +2,7 @@ import { RemoteContent } from 'nextra/data'
 import { buildDynamicMDX } from 'nextra/remote'
 import { getPageMap } from '@/src/getPageMap'
 import { remarkReplaceLinks } from '@/src/remarkReplaceLinks'
+import { remarkReplaceImages } from '@/src/remarkReplaceImages'
 import { replaceGitBookContent } from '@/src/replaceGitBookContent'
 import json from '@/remote-files/substreams.json'
 
@@ -21,24 +22,18 @@ export async function getStaticProps({ params }) {
   const baseURL = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/${docsPath}${foundPath}`
   const response = await fetch(baseURL)
   const content = await response.text()
-  const mdx = await buildDynamicMDX(
-    replaceGitBookContent({
-      content,
-      user,
-      repo,
-      branch,
-      docsPath,
-    }),
-    {
-      mdxOptions: {
-        // change-log contains `{variable}` text that is thrown an error by MDX2 parser since he treats
-        // it as variable injection, to fix it we parse chang-log with the Markdown parser
-        format: paths.endsWith('/change-log') ? 'md' : 'mdx',
-        remarkPlugins: [[remarkReplaceLinks, { foundPath, basePath: '/substreams/' }]],
-      },
-      codeHighlight: false,
-    }
-  )
+  const mdx = await buildDynamicMDX(replaceGitBookContent({ content }), {
+    mdxOptions: {
+      // change-log contains `{variable}` text that is thrown an error by MDX2 parser since he treats
+      // it as variable injection, to fix it we parse chang-log with the Markdown parser
+      format: paths.endsWith('/change-log') ? 'md' : 'mdx',
+      remarkPlugins: [
+        [remarkReplaceLinks, { foundPath, basePath: '/substreams/' }],
+        [remarkReplaceImages, { assetsBasePath: `${user}/${repo}/${branch}/${docsPath}` }],
+      ],
+    },
+    codeHighlight: false,
+  })
   return {
     props: {
       ...mdx,

website/remote-files/firehose.json

@@ -25,10 +25,10 @@
     "firehose-setup/ethereum/synchronization.md",
     "firehose-setup/near/installation.md",
     "firehose-setup/system-requirements.md",
+    "integrate-new-chains/benefits.md",
     "integrate-new-chains/design-principles.md",
     "integrate-new-chains/firehose-starter.md",
-    "integrate-new-chains/new-blockchains.md",
-    "integrate-new-chains/why-integrate-the-firehose.md",
+    "integrate-new-chains/integration-overview.md",
     "intro/firehose-overview.md",
     "intro/prerequisites.md",
     "references/faq.md",

website/route-lockfile.txt

@@ -2,7 +2,7 @@
 /ar/
 /ar/404/
 /ar/about/
-/ar/arbitrum-faq/
+/ar/arbitrum/arbitrum-faq/
 /ar/billing/
 /ar/cookbook/arweave/
 /ar/cookbook/base-testnet/
@@ -54,7 +54,7 @@
 /ar/substreams/
 /ar/tokenomics/
 /de/about/
-/de/arbitrum-faq/
+/de/arbitrum/arbitrum-faq/
 /de/billing/
 /de/cookbook/arweave/
 /de/cookbook/base-testnet/
@@ -105,7 +105,10 @@
 /en/
 /en/404/
 /en/about/
-/en/arbitrum-faq/
+/en/arbitrum-faq/ -> /en/arbitrum/arbitrum-faq/
+/en/arbitrum/arbitrum-faq/
+/en/arbitrum/l2-transfer-tools-faq/
+/en/arbitrum/l2-transfer-tools-guide/
 /en/billing/
 /en/cookbook/arweave/
 /en/cookbook/base-testnet/
@@ -149,10 +152,10 @@
 /en/firehose/firehose-setup/ethereum/synchronization/
 /en/firehose/firehose-setup/near/installation/
 /en/firehose/firehose-setup/system-requirements/
+/en/firehose/integrate-new-chains/benefits/
 /en/firehose/integrate-new-chains/design-principles/
 /en/firehose/integrate-new-chains/firehose-starter/
-/en/firehose/integrate-new-chains/new-blockchains/
-/en/firehose/integrate-new-chains/why-integrate-the-firehose/
+/en/firehose/integrate-new-chains/integration-overview/
 /en/firehose/intro/firehose-overview/
 /en/firehose/intro/prerequisites/
 /en/firehose/references/faq/
@@ -204,6 +207,7 @@
 /en/substreams/developers-guide/parallel-execution/
 /en/substreams/developers-guide/running-substreams/
 /en/substreams/developers-guide/sink-targets/README/
+/en/substreams/developers-guide/sink-targets/substreams-sink-clickhouse/
 /en/substreams/developers-guide/sink-targets/substreams-sink-files/
 /en/substreams/developers-guide/sink-targets/substreams-sink-kv/
 /en/substreams/developers-guide/sink-targets/substreams-sink-mongodb/
@@ -224,7 +228,7 @@
 /es/
 /es/404/
 /es/about/
-/es/arbitrum-faq/
+/es/arbitrum/arbitrum-faq/
 /es/billing/
 /es/cookbook/arweave/
 /es/cookbook/base-testnet/
@@ -276,7 +280,7 @@
 /es/substreams/
 /es/tokenomics/
 /fr/about/
-/fr/arbitrum-faq/
+/fr/arbitrum/arbitrum-faq/
 /fr/billing/
 /fr/cookbook/arweave/
 /fr/cookbook/base-testnet/
@@ -327,7 +331,7 @@
 /hi/
 /hi/404/
 /hi/about/
-/hi/arbitrum-faq/
+/hi/arbitrum/arbitrum-faq/
 /hi/billing/
 /hi/cookbook/arweave/
 /hi/cookbook/base-testnet/
@@ -376,7 +380,7 @@
 /hi/substreams/
 /hi/tokenomics/
 /it/about/
-/it/arbitrum-faq/
+/it/arbitrum/arbitrum-faq/
 /it/billing/
 /it/cookbook/arweave/
 /it/cookbook/base-testnet/
@@ -427,7 +431,7 @@
 /ja/
 /ja/404/
 /ja/about/
-/ja/arbitrum-faq/
+/ja/arbitrum/arbitrum-faq/
 /ja/billing/
 /ja/cookbook/arweave/
 /ja/cookbook/base-testnet/
@@ -481,7 +485,7 @@
 /ko/
 /ko/404/
 /ko/about/
-/ko/arbitrum-faq/
+/ko/arbitrum/arbitrum-faq/
 /ko/billing/
 /ko/cookbook/arweave/
 /ko/cookbook/base-testnet/
@@ -533,7 +537,7 @@
 /ko/substreams/
 /ko/tokenomics/
 /mr/about/
-/mr/arbitrum-faq/
+/mr/arbitrum/arbitrum-faq/
 /mr/billing/
 /mr/cookbook/arweave/
 /mr/cookbook/base-testnet/
@@ -582,7 +586,7 @@
 /mr/substreams/
 /mr/tokenomics/
 /nl/about/
-/nl/arbitrum-faq/
+/nl/arbitrum/arbitrum-faq/
 /nl/billing/
 /nl/cookbook/arweave/
 /nl/cookbook/base-testnet/
@@ -631,7 +635,7 @@
 /nl/substreams/
 /nl/tokenomics/
 /pl/about/
-/pl/arbitrum-faq/
+/pl/arbitrum/arbitrum-faq/
 /pl/billing/
 /pl/cookbook/arweave/
 /pl/cookbook/base-testnet/
@@ -680,7 +684,7 @@
 /pl/substreams/
 /pl/tokenomics/
 /pt/about/
-/pt/arbitrum-faq/
+/pt/arbitrum/arbitrum-faq/
 /pt/billing/
 /pt/cookbook/arweave/
 /pt/cookbook/base-testnet/
@@ -729,7 +733,7 @@
 /pt/substreams/
 /pt/tokenomics/
 /ru/about/
-/ru/arbitrum-faq/
+/ru/arbitrum/arbitrum-faq/
 /ru/billing/
 /ru/cookbook/arweave/
 /ru/cookbook/base-testnet/
@@ -778,7 +782,7 @@
 /ru/substreams/
 /ru/tokenomics/
 /sv/about/
-/sv/arbitrum-faq/
+/sv/arbitrum/arbitrum-faq/
 /sv/billing/
 /sv/cookbook/arweave/
 /sv/cookbook/base-testnet/
@@ -827,7 +831,7 @@
 /sv/substreams/
 /sv/tokenomics/
 /tr/about/
-/tr/arbitrum-faq/
+/tr/arbitrum/arbitrum-faq/
 /tr/billing/
 /tr/cookbook/arweave/
 /tr/cookbook/base-testnet/
@@ -876,7 +880,7 @@
 /tr/substreams/
 /tr/tokenomics/
 /uk/about/
-/uk/arbitrum-faq/
+/uk/arbitrum/arbitrum-faq/
 /uk/billing/
 /uk/cookbook/arweave/
 /uk/cookbook/base-testnet/
@@ -927,7 +931,7 @@
 /ur/
 /ur/404/
 /ur/about/
-/ur/arbitrum-faq/
+/ur/arbitrum/arbitrum-faq/
 /ur/billing/
 /ur/cookbook/arweave/
 /ur/cookbook/base-testnet/
@@ -979,7 +983,7 @@
 /vi/
 /vi/404/
 /vi/about/
-/vi/arbitrum-faq/
+/vi/arbitrum/arbitrum-faq/
 /vi/billing/
 /vi/cookbook/arweave/
 /vi/cookbook/base-testnet/
@@ -1033,7 +1037,7 @@
 /zh/
 /zh/404/
 /zh/about/
-/zh/arbitrum-faq/
+/zh/arbitrum/arbitrum-faq/
 /zh/billing/
 /zh/cookbook/arweave/
 /zh/cookbook/base-testnet/

website/src/remarkReplaceImages.ts

@@ -0,0 +1,33 @@
+import { Root } from 'mdast'
+import { Plugin } from 'unified'
+import { visit } from 'unist-util-visit'
+
+export const remarkReplaceImages: Plugin<[{ assetsBasePath: string }], Root> = ({ assetsBasePath }) => {
+  if (!assetsBasePath) throw new Error('remarkReplaceLinks: assetsBasePath is required')
+
+  return (tree, _file, done) => {
+    visit(
+      tree,
+      [
+        { type: 'mdxJsxTextElement', name: 'img' },
+        { type: 'mdxJsxFlowElement', name: 'img' },
+      ],
+      (node: any) => {
+        const attrs = node.attributes as { name: string; value: string }[]
+        const srcAttr = attrs.find((attr) => attr.name === 'src')!
+        if (srcAttr.value.includes('.gitbook/assets/')) {
+          srcAttr.value = srcAttr.value.replace(
+            /.*\.gitbook\/assets\//,
+            `https://raw.githubusercontent.com/${assetsBasePath}.gitbook/assets/`
+          )
+        }
+      }
+    )
+    visit(tree, 'image', (node) => {
+      if (node.url.includes('../assets/')) {
+        node.url = node.url.replace(/.*\.\.\/assets\//, `https://raw.githubusercontent.com/${assetsBasePath}assets/`)
+      }
+    })
+    done()
+  }
+}

website/src/replaceGitBookContent.ts

@@ -1,12 +1,4 @@
-type ReplaceGitBookContentParams = {
-  content: string
-  user: string
-  repo: string
-  branch: string
-  docsPath: string
-}
-
-export function replaceGitBookContent({ content, user, repo, branch, docsPath }: ReplaceGitBookContentParams): string {
+export function replaceGitBookContent({ content }: { content: string }): string {
   return (
     content
       // replace {% embed ... %} with <iframe />
@@ -23,11 +15,5 @@ export function replaceGitBookContent({ content, user, repo, branch, docsPath }:
       .replaceAll(/{%.*?%}/g, '')
       // close unclosed img tags
       .replaceAll(/<img((?:(?!\/>)[^>])*?)>/g, (...m) => `<img${m[1]}/>`)
-      // Replaces all the relative paths of images to absolute paths to the repo
-      .replaceAll('../assets', `https://raw.githubusercontent.com/${user}/${repo}/${branch}/assets/`)
-      .replaceAll(
-        '.gitbook/assets/',
-        `https://raw.githubusercontent.com/${user}/${repo}/${branch}/${docsPath}.gitbook/assets/`
-      )
   )
 }