(EAI-975) Better list of products, programming langs, topics (#760)

* Add missing products/subproducts and reorganize a bit

* Reorganize & add programming languages

* Add more topics

* dash->underscore, move some products to topics

* rename value

* Add performance advisor

* add back product that was accidentally removed

* Add name/description to topics

* Product name consistency across ingest too

* simpler way to enforce product/driver names

Author: hschawe

packages/chatbot-server-mongodb-public/src/processors/extractMongoDbMetadataFromUserMessage.eval.ts

@@ -39,7 +39,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input: "pymongo insert data",
     expected: {
       programmingLanguage: "python",
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["driver", "python"],
   },
@@ -72,7 +72,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input:
       "how to open a change stream watch on a database and filter the stream",
     expected: {
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
       programmingLanguage: "javascript",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["change_streams"],
@@ -82,7 +82,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input:
       "how to open a change stream watch on a database and filter the stream pymongo",
     expected: {
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
       programmingLanguage: "python",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["change_streams"],
@@ -134,7 +134,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input: "How do I connect to MongoDB using the Java driver?",
     expected: {
       programmingLanguage: "java",
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["driver", "java"],
   },
@@ -143,7 +143,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input: "How do I query a collection using LINQ in C#?",
     expected: {
       programmingLanguage: "csharp",
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["driver", "csharp"],
   },
@@ -161,7 +161,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input: "How do I handle MongoDB connections in Node.js?",
     expected: {
       programmingLanguage: "javascript",
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["driver", "javascript"],
   },
@@ -170,7 +170,7 @@ const evalCases: ExtractMongoDbMetadataEvalCase[] = [
     input: "How do I insert multiple documents with the MongoDB Go driver?",
     expected: {
       programmingLanguage: "go",
-      mongoDbProduct: "Drivers",
+      mongoDbProduct: "MongoDB Drivers",
     } satisfies ExtractMongoDbMetadataFunction,
     tags: ["driver", "go"],
   },

packages/chatbot-server-mongodb-public/src/processors/extractMongoDbMetadataFromUserMessage.ts

@@ -22,7 +22,7 @@ export const ExtractMongoDbMetadataFunctionSchema = z.object({
     .enum(mongoDbProductNames)
     .describe(
       `Most important MongoDB products present in the content.
-Include "Driver" if the user is asking about a programming language with a MongoDB driver.
+Include "MongoDB Drivers" if the user is asking about a programming language with a MongoDB driver.
 If the product is ambiguous, say "MongoDB Server".`
     )
     .default("MongoDB Server")
@@ -60,13 +60,13 @@ const fewShotExamples: OpenAI.Chat.ChatCompletionMessageParam[] = [
   makeUserMessage("pymongo insert data"),
   makeAssistantFunctionCallMessage(name, {
     programmingLanguage: "python",
-    mongoDbProduct: "Drivers",
+    mongoDbProduct: "MongoDB Drivers",
   } satisfies ExtractMongoDbMetadataFunction),
   // Example 5
   makeUserMessage("How do I create an index in MongoDB using the Java driver?"),
   makeAssistantFunctionCallMessage(name, {
     programmingLanguage: "java",
-    mongoDbProduct: "Drivers",
+    mongoDbProduct: "MongoDB Drivers",
   } satisfies ExtractMongoDbMetadataFunction),
   // Example 6
   makeUserMessage("$lookup"),

packages/ingest-mongodb-public/src/sources/snooty/SnootyDataSource.ts

@@ -2,6 +2,7 @@ import { createInterface } from "readline";
 import { Page, PageFormat, logger } from "mongodb-rag-core";
 import fetch from "node-fetch";
 import { DataSource, ProjectBase } from "mongodb-rag-core/dataSources";
+import { MongoDbDriverName, MongoDbProductName } from "mongodb-rag-core/mongoDbMetadata";
 import {
   snootyAstToMd,
   getTitleFromSnootyAst,
@@ -148,6 +149,7 @@ export type SnootyMetadata = {
 export type SnootyProjectConfig = ProjectBase & {
   type: "snooty";
   branches?: Branch[];
+  productName?: MongoDbDriverName | MongoDbProductName;
 };
 
 /**

packages/ingest-mongodb-public/src/sources/snooty/SnootyProjectsInfo.test.ts

@@ -6,6 +6,7 @@ import {
   prepareSnootySources,
 } from "./SnootyProjectsInfo";
 import { PageMetadata } from "mongodb-rag-core";
+import { LocallySpecifiedSnootyProjectConfig } from "./SnootyDataSource";
 
 const snootyDataApiBaseUrl = "https://snooty-data-api.mongodb.com/prod/";
 
@@ -88,7 +89,7 @@ describe("SnootyProjectsInfo", () => {
 
   describe("prepareSnootySources", () => {
     it("returns a list of Snooty data sources", async () => {
-      const projects = [
+      const projects: LocallySpecifiedSnootyProjectConfig[] = [
         {
           type: "snooty" as const,
           name: "spark-connector",
@@ -116,7 +117,7 @@ describe("SnootyProjectsInfo", () => {
       expect(page.metadata?.version).toBeDefined();
     });
     it("allows current version override", async () => {
-      const project = {
+      const project: LocallySpecifiedSnootyProjectConfig = {
         type: "snooty" as const,
         name: "spark-connector",
         tags: ["docs", "spark-connector", "spark", "apache-spark"],

packages/ingest-mongodb-public/src/sources/snootySources.ts

@@ -1,6 +1,7 @@
 import { RenderLinks } from "./snooty/snootyAstToMd";
 import { LocallySpecifiedSnootyProjectConfig } from "./snooty/SnootyDataSource";
 import { prepareSnootySources } from "./snooty/SnootyProjectsInfo";
+import { MongoDbProductName } from "mongodb-rag-core/mongoDbMetadata";
 
 export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
   {
@@ -86,13 +87,13 @@ export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
     type: "snooty",
     name: "java",
     tags: ["docs", "driver", "java", "java-sync"],
-    productName: "Java Driver",
+    productName: "Java Sync Driver",
   },
   {
     type: "snooty",
     name: "docs-k8s-operator",
     tags: ["docs", "kubernetes-operator", "kubernetes", "k8s"],
-    productName: "MongoDB Kubernetes Operator",
+    productName: "Kubernetes Operator",
   },
   {
     type: "snooty",
@@ -104,7 +105,7 @@ export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
     type: "snooty",
     name: "mck",
     tags: ["docs", "kubernetes", "k8s", "kubernetes-controllers", "kubernetes-operator"],
-    productName: "MongoDB Controllers for Kubernetes"
+    productName: "MongoDB Controllers for Kubernetes",
   },
   {
     type: "snooty",
@@ -192,8 +193,8 @@ export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
   {
     type: "snooty",
     name: "visual-studio-extension",
-    tags: ["docs", "visual-studio-extension", "visual-studio", "gui"],
-    productName: "MongoDB Visual Studio Extension",
+    tags: ["docs", "visual-studio-extension", "visual-studio", "gui", "csharp"],
+    productName: "C# Analyzer",
   },
   {
     type: "snooty",
@@ -235,7 +236,7 @@ export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
     type: "snooty",
     name: "pymongo-arrow",
     tags: ["docs", "driver", "python", "pymongo-arrow"],
-    productName: "PyMongo Arrow",
+    productName: "PyMongo Arrow Driver",
   },
   {
     type: "snooty",
@@ -265,7 +266,7 @@ export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
     type: "snooty",
     name: "atlas-architecture",
     tags: ["docs", "atlas", "architecture"],
-    productName: "MongoDB Atlas Architecture Center",
+    productName: "MongoDB Atlas Architecture Center" as MongoDbProductName,
   },
 ];