(EAI-995): add guardrail

packages/chatbot-server-mongodb-public/src/config.ts

@@ -48,6 +48,7 @@ import {
 import { useSegmentIds } from "./middleware/useSegmentIds";
 import { createAzure } from "mongodb-rag-core/aiSdk";
 import { makeSearchTool } from "./tools/search";
+import { makeMongoDbInputGuardrail } from "./processors/mongoDbInputGuardrail";
 export const {
   MONGODB_CONNECTION_URI,
   MONGODB_DATABASE_NAME,
@@ -178,6 +179,11 @@ const azureOpenAi = createAzure({
 });
 const languageModel = wrapAISDKModel(azureOpenAi("gpt-4.1"));
 
+const guardrailLanguageModel = wrapAISDKModel(azureOpenAi("gpt-4.1-mini"));
+const inputGuardrail = makeMongoDbInputGuardrail({
+  model: guardrailLanguageModel,
+});
+
 export const generateResponse = wrapTraced(
   makeVerifiedAnswerGenerateResponse({
     findVerifiedAnswer,
@@ -192,6 +198,9 @@ export const generateResponse = wrapTraced(
         languageModel,
         systemMessage: systemPrompt,
         makeReferenceLinks: makeMongoDbReferences,
+        inputGuardrail,
+        llmRefusalMessage:
+          conversations.conversationConstants.NO_RELEVANT_CONTENT,
         filterPreviousMessages: async (conversation) => {
           return conversation.messages.filter((message) => {
             return (

packages/chatbot-server-mongodb-public/src/eval/evalHelpers.ts

@@ -17,13 +17,15 @@ export const {
   OPENAI_ENDPOINT,
   OPENAI_API_VERSION,
   OPENAI_CHAT_COMPLETION_DEPLOYMENT,
+  OPENAI_RESOURCE_NAME,
 } = assertEnvVars({
   ...EVAL_ENV_VARS,
   OPENAI_CHAT_COMPLETION_DEPLOYMENT: "",
   OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: "",
   OPENAI_API_KEY: "",
   OPENAI_ENDPOINT: "",
   OPENAI_API_VERSION: "",
+  OPENAI_RESOURCE_NAME: "",
 });
 
 export const openAiClient = new AzureOpenAI({

packages/chatbot-server-mongodb-public/src/processors/makeMongoDbReferences.test.ts

@@ -66,32 +66,37 @@ describe("makeMongoDbReferences", () => {
         chunkIndex: 0,
       },
     ] satisfies EmbeddedContent[];
-    const result = makeMongoDbReferences(chunks);
+    const result = makeMongoDbReferences(
+      chunks.map((c) => ({
+        ...c,
+        title: c.metadata?.pageTitle,
+      }))
+    );
     expect(result).toEqual([
       {
         url: "https://www.example.com/blog",
         title: "Example Blog",
         metadata: {
-          sourceName: "example",
           sourceType: "Blog",
+          sourceName: "example",
           tags: ["external", "example"],
         },
       },
       {
         url: "https://www.mongodb.com/love-your-developers",
         title: "Love Your Developers",
         metadata: {
-          sourceName: "mongodb-dotcom",
           sourceType: "Article",
+          sourceName: "mongodb-dotcom",
           tags: ["external", "example"],
         },
       },
       {
         url: "https://www.mongodb.com/developer/products/mongodb/best-practices-flask-mongodb",
         title: "Best Practices for Using Flask and MongoDB",
         metadata: {
-          sourceName: "devcenter",
           sourceType: "Article",
+          sourceName: "devcenter",
           tags: ["devcenter", "example", "python", "flask"],
         },
       },
@@ -114,7 +119,12 @@ describe("makeMongoDbReferences", () => {
         chunkIndex: 0,
       },
     ];
-    const result = makeMongoDbReferences(chunks);
+    const result = makeMongoDbReferences(
+      chunks.map((c) => ({
+        ...c,
+        title: c.metadata?.pageTitle,
+      }))
+    );
     expect(result).toEqual([
       {
         url: "https://www.example.com/somepage",
@@ -134,13 +144,11 @@ describe("addReferenceSourceType", () => {
       url: "https://mongodb.com/docs/manual/reference/operator/query/eq/",
       title: "$eq",
       metadata: {
-        sourceName: "snooty-docs",
         tags: ["docs", "manual"],
       },
     };
     const result = addReferenceSourceType(reference);
     expect(result.metadata).toEqual({
-      sourceName: reference.metadata?.sourceName,
       tags: reference.metadata?.tags,
       sourceType: "Docs",
     });
@@ -151,7 +159,6 @@ describe("addReferenceSourceType", () => {
       url: "https://mongodb.com/docs/manual/reference/operator/query/eq/",
       title: "$eq",
       metadata: {
-        sourceName: "snooty-docs",
         sourceType: "ThinAir",
         tags: ["docs", "manual"],
       },
@@ -169,7 +176,6 @@ describe("addReferenceSourceType", () => {
       url: "https://example.com/random-thoughts/hotdogs-are-tacos",
       title: "Hotdogs Are Just Tacos",
       metadata: {
-        sourceName: "some-random-blog",
         tags: ["external"],
       },
     };