openai: improve result parsing

src/ext/openai/openai.ts

@@ -8,6 +8,7 @@ import {
 import { createLogger } from '@/logger';
 import { type EvaluationSummaryInput } from '@/service/EvaluationService';
 import { env } from '@/env';
+import { parseArray, parseObject } from '@/utils';
 
 const openai = new OpenAI({
   apiKey: env.OPENAI_API_KEY,
@@ -54,7 +55,7 @@ export const requestEvaluation = async (
   );
   const result = await queryOpenAI(prompt);
   logger.info('Application evaluation complete', { result });
-  return JSON.parse(removeJsonCodeBlocks(result));
+  return parseObject(result) as EvaluationSummaryInput;
 };
 
 export const requestEvaluationQuestions = async (
@@ -64,11 +65,5 @@ export const requestEvaluationQuestions = async (
   const prompt: string = createEvaluationQuestionPrompt(roundMetadata);
   const result = await queryOpenAI(prompt);
   logger.info('Received evaluation questions from OpenAI', { result });
-  return JSON.parse(removeJsonCodeBlocks(result)).map(line =>
-    line.replace(/^\d+\.\s*/, '').trim()
-  );
-};
-
-const removeJsonCodeBlocks = (str: string): string => {
-  return str.replace(/```json/g, '').replace(/```/g, '');
-};
+  return parseArray(result).map(line => line.replace(/^\d+\.\s*/, '').trim());
+};

src/ext/openai/prompt.ts

@@ -67,7 +67,7 @@ export const createAiEvaluationPrompt = (
   ${questionsString}
 
   Your question answers should NOT be 'yes', 'no', or 'uncertain'. Please use always 0 for 'yes', 1 for 'no', and 2 for 'uncertain'.
-    Please respond with ONLY the following JSON structure and NOTHING else:
+    Please respond with a valid JSON structure and NOTHING else:
     {
       "questions": [
         {

src/utils.ts

@@ -78,3 +78,42 @@ export async function isPoolManager<T>(
     return false;
   }
 }
+
+export function parseObject(s): object {
+  const jsonObjPattern = /\{(?:[^{}]*|(?:\{(?:[^{}]*|(?:\{[^{}]*\}))*\}))*\}/g;
+
+  // Find all matches for JSON objects
+  const jsonObjs = s.match(jsonObjPattern) ?? [];
+
+  // Parse the matches
+  return jsonObjs
+    .map(obj => {
+      try {
+        return JSON.parse(obj as string);
+      } catch (e) {
+        console.error('Failed to parse JSON object:', obj);
+        return null;
+      }
+    })
+    .filter(obj => obj !== null)[0];
+}
+
+export function parseArray(s): string[] {
+  // Regular expression to find JSON arrays
+  const jsonArrayPattern = /\[[^[\]]*\]/g;
+
+  // Find all matches for JSON arrays
+  const jsonArrays = s.match(jsonArrayPattern) ?? [];
+
+  // Parse the matches
+  return jsonArrays
+    .map(arr => {
+      try {
+        return JSON.parse(arr as string);
+      } catch (e) {
+        console.error('Failed to parse JSON array:', arr);
+        return null;
+      }
+    })
+    .filter(arr => arr !== null)[0];
+}