diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index 8092967..80451d7 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -300,8 +300,9 @@ const batchPromises = (array: T[], batchSize: number): T[][] => { export const createLLMEvaluations = async ( paramsArray: CreateLLMEvaluationParams[] -): Promise => { +): Promise => { const roundCache: Record = {}; + const failedProjects: string[] = []; // Split the paramsArray into batches of 10 const batchedParams = batchPromises(paramsArray, 10); @@ -310,86 +311,95 @@ export const createLLMEvaluations = async ( try { // Process each batch of promises concurrently const evaluationPromises = batch.map(async params => { - const evaluationQuestions = - params.questions === undefined || params.questions.length === 0 - ? await evaluationService.getQuestionsByChainAndAlloPoolId( - params.chainId, - params.alloPoolId - ) - : params.questions; - - if (evaluationQuestions === null || evaluationQuestions.length === 0) { - logger.error( - 'createLLMEvaluations:Failed to get evaluation questions' - ); - throw new Error('Failed to get evaluation questions'); - } - - let roundMetadata = params.roundMetadata; - let applicationMetadata = params.applicationMetadata; - - // Check if the round is already in cache - if (roundMetadata == null || applicationMetadata == null) { - let round: RoundWithApplications | null; - - // If the round is cached, use it - if (roundCache[params.alloPoolId] != null) { - round = roundCache[params.alloPoolId]; - logger.debug( - `Using cached round data for roundId: ${params.alloPoolId}` - ); - } else { - // Fetch the round and store it in the cache - const [error, fetchedRound] = await catchError( - indexerClient.getRoundWithApplications({ - chainId: params.chainId, - roundId: params.alloPoolId, - }) + try { + const evaluationQuestions = + params.questions === undefined || params.questions.length === 0 + ? await evaluationService.getQuestionsByChainAndAlloPoolId( + params.chainId, + params.alloPoolId + ) + : params.questions; + + if ( + evaluationQuestions === null || + evaluationQuestions.length === 0 + ) { + logger.error( + 'createLLMEvaluations:Failed to get evaluation questions' ); + throw new Error('Failed to get evaluation questions'); + } - if (error !== undefined || fetchedRound == null) { - logger.error('Failed to fetch round with applications'); - throw new Error('Failed to fetch round with applications'); + let roundMetadata = params.roundMetadata; + let applicationMetadata = params.applicationMetadata; + + // Check if the round is already in cache + if (roundMetadata == null || applicationMetadata == null) { + let round: RoundWithApplications | null; + + // If the round is cached, use it + if (roundCache[params.alloPoolId] != null) { + round = roundCache[params.alloPoolId]; + logger.debug( + `Using cached round data for roundId: ${params.alloPoolId}` + ); + } else { + // Fetch the round and store it in the cache + const [error, fetchedRound] = await catchError( + indexerClient.getRoundWithApplications({ + chainId: params.chainId, + roundId: params.alloPoolId, + }) + ); + + if (error !== undefined || fetchedRound == null) { + logger.error('Failed to fetch round with applications'); + throw new Error('Failed to fetch round with applications'); + } + + round = fetchedRound; + roundCache[params.alloPoolId] = round; + logger.info( + `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + ); } - round = fetchedRound; - roundCache[params.alloPoolId] = round; - logger.info( - `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + const application = round.applications.find( + app => app.id === params.alloApplicationId ); + if (application == null) { + logger.error( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + throw new NotFoundError( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + } + + roundMetadata = round.roundMetadata; + applicationMetadata = application.metadata; } - const application = round.applications.find( - app => app.id === params.alloApplicationId + const evaluation = await requestEvaluation( + roundMetadata, + applicationMetadata, + evaluationQuestions ); - if (application == null) { - logger.error( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - throw new NotFoundError( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - } - roundMetadata = round.roundMetadata; - applicationMetadata = application.metadata; + await createEvaluation({ + chainId: params.chainId, + alloPoolId: params.alloPoolId, + alloApplicationId: params.alloApplicationId, + cid: params.cid, + evaluator: params.evaluator, + summaryInput: evaluation, + evaluatorType: EVALUATOR_TYPE.LLM_GPT3, + }); + } catch (error) { + // If an error occurs, add the project ID to the failedProjects array + failedProjects.push(params.alloApplicationId); + throw error; } - - const evaluation = await requestEvaluation( - roundMetadata, - applicationMetadata, - evaluationQuestions - ); - - await createEvaluation({ - chainId: params.chainId, - alloPoolId: params.alloPoolId, - alloApplicationId: params.alloApplicationId, - cid: params.cid, - evaluator: params.evaluator, - summaryInput: evaluation, - evaluatorType: EVALUATOR_TYPE.LLM_GPT3, - }); }); await Promise.all(evaluationPromises); @@ -405,4 +415,6 @@ export const createLLMEvaluations = async ( continue; } } + + return failedProjects; }; diff --git a/src/controllers/poolController.ts b/src/controllers/poolController.ts index 66faf38..b740631 100644 --- a/src/controllers/poolController.ts +++ b/src/controllers/poolController.ts @@ -101,16 +101,34 @@ export const syncPool = async (req: Request, res: Response): Promise => { // ---- LLM evaluation ---- // Trigger LLM evaluation for the pool if there are applications without evaluations - if (skipEvaluation !== true) - await triggerLLMEvaluationByPool( + let failedProjects: string[] = []; + if (skipEvaluation !== true) { + failedProjects = await triggerLLMEvaluationByPool( alloPoolId, chainId, indexerPoolData, evaluationQuestions ); - // Log success and respond to the request - logger.info('successfully synced pool', pool); - res.status(200).json({ message: 'pool synced successfully' }); + } + + // Check if there are any failed projects and respond accordingly + if (failedProjects.length > 0) { + logger.info('Pool synced successfully with some projects failing', { + pool, + failedProjects, + }); + res.status(207).json({ + success: true, + message: 'Pool synced successfully, with some projects failing.', + failedProjects, + }); + } else { + logger.info('Successfully synced pool', pool); + res.status(200).json({ + success: true, + message: 'Pool synced successfully.', + }); + } }; const handlePoolEvaluationQuestions = async ( @@ -171,7 +189,7 @@ const triggerLLMEvaluationByPool = async ( chainId: number, indexerPoolData: IndexerRoundWithApplications, evaluationQuestions: PromptEvaluationQuestions -): Promise => { +): Promise => { const applicationsWithoutLLM = await applicationService.getApplicationsWithoutLLMEvalutionsByAlloPoolId( alloPoolId, @@ -212,5 +230,5 @@ const triggerLLMEvaluationByPool = async ( logger.warn('Some applications were not found in indexerPoolData'); } - await createLLMEvaluations(evaluationParamsArray); + return await createLLMEvaluations(evaluationParamsArray); }; diff --git a/src/ext/openai/prompt.ts b/src/ext/openai/prompt.ts index f6ad5fc..a335f13 100644 --- a/src/ext/openai/prompt.ts +++ b/src/ext/openai/prompt.ts @@ -39,11 +39,11 @@ const sanitizeAndReduceMetadata = ( }; const sanitizeRoundMetadata = (metadata: RoundMetadata): string => { - return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 1000); + return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 50000); }; const sanitizeApplicationMetadata = (metadata: object): string => { - return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 3000); + return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 50000); }; export const createAiEvaluationPrompt = ( roundMetadata: RoundMetadata, @@ -52,7 +52,7 @@ export const createAiEvaluationPrompt = ( ): string => { const sanitizedRoundMetadata = sanitizeRoundMetadata(roundMetadata); const sanitizedApplicationMetadata = sanitizeApplicationMetadata( - applicationMetadata.application.project + applicationMetadata.application ); const questionsString = applicationQuestions diff --git a/src/routes/poolRoutes.ts b/src/routes/poolRoutes.ts index cd1e51c..0090488 100644 --- a/src/routes/poolRoutes.ts +++ b/src/routes/poolRoutes.ts @@ -18,31 +18,43 @@ const router = Router(); * alloPoolId: * type: string * description: The ID of the pool to create - * example: "609" # Example of poolId + * example: "609" * chainId: * type: number * description: The chain ID associated with the pool - * example: 42161 # Example of chainId (Arbitrum) + * example: 42161 * skipEvaluation: - * type: boolean - * description: Skip evaluation of the pool - * example: true + * type: boolean + * description: Skip evaluation of the pool + * example: true * required: * - alloPoolId * - chainId * responses: * 201: - * description: Pool created successfully + * description: Pool synced successfully + * 207: + * description: Pool synced successfully, with some projects failing + * content: + * application/json: + * schema: + * type: object + * properties: + * success: + * type: boolean + * example: true + * message: + * type: string + * example: Pool synced successfully, with some projects failing. + * failedProjects: + * type: array + * items: + * type: string + * example: "0xprojectId1" * 400: * description: Invalid poolId or chainId format * 500: * description: Internal server error - * examples: - * application/json: - * - value: - * alloPoolId: "609" - * chainId: "42161" - * skipEvaluation: true */ router.post('/', syncPool);