(EAI-1033): include TOC index for snooty ingest (#734)

* include TOC index

* fix ai gen bad inclusion

---------

Co-authored-by: Ben Perlmutter <mongodben@mongodb.com>

Author: mongodben

packages/ingest-mongodb-public/src/sources/snooty/SnootyDataSource.test.ts

@@ -32,6 +32,7 @@ describe("SnootyDataSource", () => {
     ],
   };
   const snootyDataApiBaseUrl = "https://snooty-data-api.mongodb.com/prod/";
+
   describe("makeSnootyDataSource()", () => {
     const sampleDataPath = Path.resolve(
       SRC_ROOT,
@@ -251,6 +252,17 @@ describe("SnootyDataSource", () => {
       expect(pages).toHaveLength(1);
       noIndexMock.done();
     });
+
+    it("includes tocIndex in metadata", async () => {
+      const source = await makeSnootyDataSource({
+        name: `snooty-test`,
+        project,
+        snootyDataApiBaseUrl,
+      });
+      const pages = await source.fetchPages();
+
+      expect(pages[0].metadata?.page?.tocIndex).toBe(0);
+    });
   });
 });
 
@@ -270,6 +282,7 @@ describe("handlePage()", () => {
       baseUrl: "https://example.com",
       tags: ["a"],
       version: { label: "1.0", isCurrent: true },
+      toc: [],
     });
     expect(result).toMatchObject({
       format: "openapi-yaml",
@@ -292,6 +305,7 @@ describe("handlePage()", () => {
       baseUrl: "https://example.com",
       tags: ["a"],
       version: { label: "1.0", isCurrent: true },
+      toc: [],
     });
     expect(result).toMatchObject({
       format: "md",

packages/ingest-mongodb-public/src/sources/snooty/SnootyDataSource.ts

@@ -28,12 +28,30 @@ export type SnootyPageEntry = SnootyManifestEntry & {
   data: SnootyPageData;
 };
 
+export interface SnootyTocEntry {
+  title: {
+    type: "text";
+    position: {
+      start: {
+        line: number;
+      };
+    };
+    value: string;
+  }[];
+  slug?: string;
+  url?: string;
+  children: SnootyTocEntry[];
+  options?: {
+    drawer?: boolean;
+  };
+}
+
 /**
   Represents metadata in a Snooty manifest file.
  */
 export type SnootyMetadataEntry = SnootyManifestEntry & {
   type: "metadata";
-  data: { title?: string };
+  data: { title?: string; toctree: SnootyTocEntry; toctreeOrder: string[] };
 };
 
 /**
@@ -202,6 +220,7 @@ export const makeSnootyDataSource = ({
         const linePromises: Promise<void>[] = [];
         const pages: Page[] = [];
         let siteTitle: string | undefined = undefined;
+        const toc: string[] = [];
         await new Promise<void>((resolve, reject) => {
           stream.on("line", async (line) => {
             const entry = JSON.parse(line) as SnootyManifestEntry;
@@ -227,6 +246,7 @@ export const makeSnootyDataSource = ({
                           ...links,
                           baseUrl: branchUrl,
                         },
+                        toc,
                       });
                       if (page !== undefined) {
                         pages.push(page);
@@ -249,6 +269,18 @@ export const makeSnootyDataSource = ({
               case "metadata": {
                 const { data } = entry as SnootyMetadataEntry;
                 siteTitle = data.title;
+                const visitedUrls = new Set<string>();
+                const tocUrls = data.toctreeOrder
+                  .filter((slug) => {
+                    const url = makeUrl(slug, branchUrl);
+                    if (visitedUrls.has(url)) {
+                      return false;
+                    }
+                    visitedUrls.add(url);
+                    return true;
+                  })
+                  .map((slug) => makeUrl(slug, branchUrl));
+                toc.push(...tocUrls);
                 return;
               }
               case "timestamp":
@@ -344,6 +376,7 @@ export const handlePage = async (
     productName,
     version,
     links,
+    toc,
   }: {
     sourceName: string;
     baseUrl: string;
@@ -354,6 +387,7 @@ export const handlePage = async (
       isCurrent: boolean;
     };
     links?: RenderLinks;
+    toc: string[];
   }
 ): Promise<Page | undefined> => {
   // Strip first three path segments - according to Snooty team, they'll always
@@ -373,11 +407,8 @@ export const handlePage = async (
   let body = "";
   let title: string | undefined;
   let format: PageFormat;
-  const baseUrlTrailingSlash = baseUrl.replace(/\/?$/, "/");
-  const url = new URL(pagePath, baseUrlTrailingSlash).href.replace(
-    /\/?$/, // Add trailing slash
-    "/"
-  );
+  const url = makeUrl(pagePath, baseUrl);
+
   if (page.ast.options?.template === "openapi") {
     format = "openapi-yaml";
     body = await snootyAstToOpenApiSpec(page.ast);
@@ -395,6 +426,9 @@ export const handlePage = async (
     return;
   }
 
+  const maybeTocIndex = toc.findIndex((tocUrl) => tocUrl === url);
+  const tocIndex = maybeTocIndex === -1 ? undefined : maybeTocIndex;
+
   return {
     url,
     sourceName,
@@ -403,10 +437,28 @@ export const handlePage = async (
     format,
     sourceType: "tech-docs",
     metadata: {
-      page: pageMetadata,
+      page: { ...pageMetadata, tocIndex },
       tags,
       productName,
       version,
     },
   };
 };
+
+function makeUrl(pagePath: string, baseUrl: string): string {
+  // Ensure trailing slash for baseUrl
+  const baseUrlTrailingSlash = baseUrl.replace(/\/?$/, "/");
+
+  // Handle empty pagePath or root path
+  if (!pagePath || pagePath === "/") {
+    return baseUrlTrailingSlash;
+  }
+
+  // For non-empty paths, remove leading slash and ensure trailing slash
+  const cleanPagePath = pagePath
+    .replace(/^\//, "") // Remove leading slash
+    .replace(/\/?$/, "/"); // Ensure trailing slash
+
+  // Concatenate the base URL with the clean page path
+  return baseUrlTrailingSlash + cleanPagePath;
+}