diff --git a/src/ext/openai/openai.ts b/src/ext/openai/openai.ts index f89453c..fba9881 100644 --- a/src/ext/openai/openai.ts +++ b/src/ext/openai/openai.ts @@ -8,6 +8,7 @@ import { import { createLogger } from '@/logger'; import { type EvaluationSummaryInput } from '@/service/EvaluationService'; import { env } from '@/env'; +import { parseArray, parseObject } from '@/utils'; const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY, @@ -54,7 +55,7 @@ export const requestEvaluation = async ( ); const result = await queryOpenAI(prompt); logger.info('Application evaluation complete', { result }); - return JSON.parse(removeJsonCodeBlocks(result)); + return parseObject(result) as EvaluationSummaryInput; }; export const requestEvaluationQuestions = async ( @@ -64,11 +65,5 @@ export const requestEvaluationQuestions = async ( const prompt: string = createEvaluationQuestionPrompt(roundMetadata); const result = await queryOpenAI(prompt); logger.info('Received evaluation questions from OpenAI', { result }); - return JSON.parse(removeJsonCodeBlocks(result)).map(line => - line.replace(/^\d+\.\s*/, '').trim() - ); -}; - -const removeJsonCodeBlocks = (str: string): string => { - return str.replace(/```json/g, '').replace(/```/g, ''); + return parseArray(result).map(line => line.replace(/^\d+\.\s*/, '').trim()); }; diff --git a/src/ext/openai/prompt.ts b/src/ext/openai/prompt.ts index 074c426..f6ad5fc 100644 --- a/src/ext/openai/prompt.ts +++ b/src/ext/openai/prompt.ts @@ -67,7 +67,7 @@ export const createAiEvaluationPrompt = ( ${questionsString} Your question answers should NOT be 'yes', 'no', or 'uncertain'. Please use always 0 for 'yes', 1 for 'no', and 2 for 'uncertain'. - Please respond with ONLY the following JSON structure and NOTHING else: + Please respond with a valid JSON structure and NOTHING else: { "questions": [ { @@ -84,7 +84,6 @@ export const createAiEvaluationPrompt = ( export const createEvaluationQuestionPrompt = ( roundMetadata: RoundMetadata ): string => { - console.log('roundMetadata', roundMetadata); return `Given the following description of a Grants round, generate 5 evaluation questions that a reviewer can answer with 'Yes', 'No', or 'Uncertain'. The questions should help assess the projects in this round, focusing on the following key aspects: Ensure that each question is focused on one specific aspect of evaluation, is clear and concise, and can be answered with 'Yes', 'No', or 'Uncertain'. Also, avoid overly simplistic or binary questions unless they are critical for assessing eligibility. The questions should help a reviewer evaluate the project in a thoughtful, critical, and fair manner. diff --git a/src/utils.ts b/src/utils.ts index 0b2c5d4..c817a72 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -78,3 +78,36 @@ export async function isPoolManager( return false; } } + +function extractAndParseJSON(s: string, pattern: RegExp): any[] { + const matches = s.match(pattern) ?? []; + + const parsedMatches = matches + .map(match => { + try { + return JSON.parse(match as string); + } catch (e) { + console.error('Failed to parse JSON:', match); + return null; + } + }) + .filter(parsed => parsed !== null); + + if (parsedMatches.length === 0) { + throw new Error('No valid JSON found'); + } + + return parsedMatches; +} + +export function parseObject(s: string): object { + const jsonObjPattern = /\{(?:[^{}]*|(?:\{(?:[^{}]*|(?:\{[^{}]*\}))*\}))*\}/g; + const parsedObjects = extractAndParseJSON(s, jsonObjPattern); + return parsedObjects[0]; +} + +export function parseArray(s: string): any[] { + const jsonArrayPattern = /\[[^[\]]*\]/g; + const parsedArrays = extractAndParseJSON(s, jsonArrayPattern); + return parsedArrays[0]; +}