openai: improve result parsing

src/ext/openai/openai.ts

@@ -8,6 +8,7 @@ import {
 import { createLogger } from '@/logger';
 import { type EvaluationSummaryInput } from '@/service/EvaluationService';
 import { env } from '@/env';
+import { parseArray, parseObject } from '@/utils';
 
 const openai = new OpenAI({
   apiKey: env.OPENAI_API_KEY,
@@ -54,7 +55,7 @@ export const requestEvaluation = async (
   );
   const result = await queryOpenAI(prompt);
   logger.info('Application evaluation complete', { result });
-  return JSON.parse(removeJsonCodeBlocks(result));
+  return parseObject(result) as EvaluationSummaryInput;
 };
 
 export const requestEvaluationQuestions = async (
@@ -64,11 +65,5 @@ export const requestEvaluationQuestions = async (
   const prompt: string = createEvaluationQuestionPrompt(roundMetadata);
   const result = await queryOpenAI(prompt);
   logger.info('Received evaluation questions from OpenAI', { result });
-  return JSON.parse(removeJsonCodeBlocks(result)).map(line =>
-    line.replace(/^\d+\.\s*/, '').trim()
-  );
-};
-
-const removeJsonCodeBlocks = (str: string): string => {
-  return str.replace(/```json/g, '').replace(/```/g, '');
+  return parseArray(result).map(line => line.replace(/^\d+\.\s*/, '').trim());
 };

src/ext/openai/prompt.ts

@@ -67,7 +67,7 @@ export const createAiEvaluationPrompt = (
   ${questionsString}
 
   Your question answers should NOT be 'yes', 'no', or 'uncertain'. Please use always 0 for 'yes', 1 for 'no', and 2 for 'uncertain'.
-    Please respond with ONLY the following JSON structure and NOTHING else:
+    Please respond with a valid JSON structure and NOTHING else:
     {
       "questions": [
         {
@@ -84,7 +84,6 @@ export const createAiEvaluationPrompt = (
 export const createEvaluationQuestionPrompt = (
   roundMetadata: RoundMetadata
 ): string => {
-  console.log('roundMetadata', roundMetadata);
   return `Given the following description of a Grants round, generate 5 evaluation questions that a reviewer can answer with 'Yes', 'No', or 'Uncertain'. The questions should help assess the projects in this round, focusing on the following key aspects:
 
     Ensure that each question is focused on one specific aspect of evaluation, is clear and concise, and can be answered with 'Yes', 'No', or 'Uncertain'. Also, avoid overly simplistic or binary questions unless they are critical for assessing eligibility. The questions should help a reviewer evaluate the project in a thoughtful, critical, and fair manner.

src/utils.ts

@@ -78,3 +78,36 @@ export async function isPoolManager<T>(
     return false;
   }
 }
+
+function extractAndParseJSON(s: string, pattern: RegExp): any[] {
+  const matches = s.match(pattern) ?? [];
+
+  const parsedMatches = matches
+    .map(match => {
+      try {
+        return JSON.parse(match as string);
+      } catch (e) {
+        console.error('Failed to parse JSON:', match);
+        return null;
+      }
+    })
+    .filter(parsed => parsed !== null);
+
+  if (parsedMatches.length === 0) {
+    throw new Error('No valid JSON found');
+  }
+
+  return parsedMatches;
+}
+
+export function parseObject(s: string): object {
+  const jsonObjPattern = /\{(?:[^{}]*|(?:\{(?:[^{}]*|(?:\{[^{}]*\}))*\}))*\}/g;
+  const parsedObjects = extractAndParseJSON(s, jsonObjPattern);
+  return parsedObjects[0];
+}
+
+export function parseArray(s: string): any[] {
+  const jsonArrayPattern = /\[[^[\]]*\]/g;
+  const parsedArrays = extractAndParseJSON(s, jsonArrayPattern);
+  return parsedArrays[0];
+}