(EAI-895): Remove AnswerRelevancy tracing metric (#686)

remove AnswerRelevancy

Co-authored-by: Ben Perlmutter <mongodben@mongodb.com>

Author: mongodben

packages/chatbot-server-mongodb-public/src/tracing/getLlmAsAJudgeScores.test.ts

@@ -8,9 +8,6 @@ jest.mock("autoevals", () => ({
   ContextRelevancy: jest.fn().mockResolvedValue({
     score: 0.8,
   }),
-  AnswerRelevancy: jest.fn().mockResolvedValue({
-    score: 0.8,
-  }),
 }));
 
 afterEach(() => {
@@ -100,7 +97,6 @@ describe("getLlmAsAJudgeScores", () => {
     const scores = await getLlmAsAJudgeScores(fakeBaseConfig, willJudge);
     expect(scores).toEqual({
       ContextRelevancy: 0.8,
-      AnswerRelevancy: 0.8,
       Faithfulness: 0.8,
     });
   });

packages/chatbot-server-mongodb-public/src/tracing/getLlmAsAJudgeScores.ts

@@ -31,13 +31,7 @@ const makeEvaluateWithLlmAsAJudge = (
   openAiConfig: LlmAsAJudge["openAiConfig"]
 ) =>
   wrapTraced(
-    async function ({
-      input,
-      output,
-      context,
-      judgeEmbeddingModel,
-      judgeModel,
-    }: ScorerArgs) {
+    async function ({ input, output, context, judgeModel }: ScorerArgs) {
       return Promise.all([
         traced(
           async () =>
@@ -52,20 +46,6 @@ const makeEvaluateWithLlmAsAJudge = (
             name: "Faithfulness",
           }
         ),
-        traced(
-          async () =>
-            AnswerRelevancy({
-              input,
-              output,
-              context,
-              model: judgeModel,
-              embeddingModel: judgeEmbeddingModel,
-              ...openAiConfig,
-            }),
-          {
-            name: "AnswerRelevancy",
-          }
-        ),
         traced(
           async () =>
             ContextRelevancy({
@@ -113,19 +93,18 @@ export async function getLlmAsAJudgeScores(
 
   const evaluateWithLlmAsAJudge = makeEvaluateWithLlmAsAJudge(openAiConfig);
 
-  const [faithfulness, answerRelevancy, contextRelevancy] = context
+  const [faithfulness, contextRelevancy] = context
     ? await evaluateWithLlmAsAJudge({
         input,
         output,
         context,
         judgeModel,
         judgeEmbeddingModel,
       })
-    : [nullScore, nullScore, nullScore];
+    : [nullScore, nullScore];
 
   return {
     Faithfulness: faithfulness.score,
-    AnswerRelevancy: answerRelevancy.score,
     ContextRelevancy: contextRelevancy.score,
   };
 }