diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index f8046f7..19b32fd 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -14,6 +14,7 @@ import poolService from '@/service/PoolService'; import { type PromptEvaluationQuestions, requestEvaluation, + requestEvaluationQuestions, } from '@/ext/openai'; import { type ApplicationMetadata, @@ -24,6 +25,13 @@ import { import { type Evaluation, EVALUATOR_TYPE } from '@/entity/Evaluation'; import { IsNullError, NotFoundError } from '@/errors'; import { type Hex } from 'viem'; +import type { + PoolIdChainId, + PoolIdChainIdApplicationId, + PoolIdChainIdApplicationIdBody, + PoolIdChainIdBody, +} from './types'; +import evaluationQuestionService from '@/service/EvaluationQuestionService'; const logger = createLogger(); @@ -31,6 +39,67 @@ interface EvaluationBody extends CreateEvaluationParams { signature: Hex; } +export const recreateEvaluationQuestions = async ( + req: Request, + res: Response +): Promise => { + const { chainId, alloPoolId, signature } = req.body as PoolIdChainIdBody; + + const isAllowed = await isPoolManager( + { alloPoolId, chainId }, + signature, + chainId, + alloPoolId + ); + + if (!isAllowed) { + logger.warn( + `User with address: ${signature} is not allowed to evaluate application` + ); + res.status(403).json({ message: 'Unauthorized' }); + return; + } + + const [error, roundWithApplications] = await catchError( + indexerClient.getRoundWithApplications({ + chainId, + roundId: alloPoolId, + }) + ); + + if ( + error !== undefined || + roundWithApplications === undefined || + roundWithApplications?.roundMetadata === undefined + ) { + logger.error('Failed to fetch round with applications'); + res + .status(404) + .json({ message: 'Failed to fetch round with applications' }); + return; + } + + const evaluationQuestions = await requestEvaluationQuestions( + roundWithApplications.roundMetadata + ); + + if (evaluationQuestions === null || evaluationQuestions.length === 0) { + logger.error('Failed to get evaluation questions'); + res.status(404).json({ message: 'Failed to get evaluation questions' }); + return; + } + + await evaluationQuestionService.resetEvaluationQuestions( + chainId, + alloPoolId, + evaluationQuestions + ); + + await evaluationService.cleanEvaluations(); + + res.status(200).json(evaluationQuestions); +}; + export const evaluateApplication = async ( req: Request, res: Response @@ -151,16 +220,6 @@ export interface CreateLLMEvaluationParams { applicationMetadata?: ApplicationMetadata; questions?: PromptEvaluationQuestions; } -interface PoolIdChainIdApplicationId { - alloPoolId: string; - chainId: number; - alloApplicationId: string; -} - -interface PoolIdChainIdApplicationIdBody extends PoolIdChainIdApplicationId { - signature: Hex; -} - export const triggerLLMEvaluation = async ( req: Request, res: Response @@ -229,91 +288,119 @@ export const triggerLLMEvaluation = async ( } }; +const batchPromises = (array: T[], batchSize: number): T[][] => { + const batches: T[][] = []; + for (let i = 0; i < array.length; i += batchSize) { + batches.push(array.slice(i, i + batchSize)); + } + return batches; +}; + export const createLLMEvaluations = async ( paramsArray: CreateLLMEvaluationParams[] ): Promise => { const roundCache: Record = {}; - const evaluationPromises = paramsArray.map(async params => { - const evaluationQuestions = - params.questions === undefined || params.questions.length === 0 - ? await evaluationService.getQuestionsByChainAndAlloPoolId( - params.chainId, - params.alloPoolId - ) - : params.questions; - - if (evaluationQuestions === null || evaluationQuestions.length === 0) { - logger.error('createLLMEvaluations:Failed to get evaluation questions'); - throw new Error('Failed to get evaluation questions'); - } - - let roundMetadata = params.roundMetadata; - let applicationMetadata = params.applicationMetadata; - // Check if the round is already in cache - if (roundMetadata == null || applicationMetadata == null) { - let round: RoundWithApplications | null; - - // If the round is cached, use it - if (roundCache[params.alloPoolId] != null) { - round = roundCache[params.alloPoolId]; - logger.debug( - `Using cached round data for roundId: ${params.alloPoolId}` - ); - } else { - // Fetch the round and store it in the cache - const [error, fetchedRound] = await catchError( - indexerClient.getRoundWithApplications({ - chainId: params.chainId, - roundId: params.alloPoolId, - }) - ); + // Split the paramsArray into batches of 10 + const batchedParams = batchPromises(paramsArray, 10); + + for (const batch of batchedParams) { + try { + // Process each batch of promises concurrently + const evaluationPromises = batch.map(async params => { + const evaluationQuestions = + params.questions === undefined || params.questions.length === 0 + ? await evaluationService.getQuestionsByChainAndAlloPoolId( + params.chainId, + params.alloPoolId + ) + : params.questions; + + if (evaluationQuestions === null || evaluationQuestions.length === 0) { + logger.error( + 'createLLMEvaluations:Failed to get evaluation questions' + ); + throw new Error('Failed to get evaluation questions'); + } - if (error !== undefined || fetchedRound == null) { - logger.error('Failed to fetch round with applications'); - throw new Error('Failed to fetch round with applications'); + let roundMetadata = params.roundMetadata; + let applicationMetadata = params.applicationMetadata; + + // Check if the round is already in cache + if (roundMetadata == null || applicationMetadata == null) { + let round: RoundWithApplications | null; + + // If the round is cached, use it + if (roundCache[params.alloPoolId] != null) { + round = roundCache[params.alloPoolId]; + logger.debug( + `Using cached round data for roundId: ${params.alloPoolId}` + ); + } else { + // Fetch the round and store it in the cache + const [error, fetchedRound] = await catchError( + indexerClient.getRoundWithApplications({ + chainId: params.chainId, + roundId: params.alloPoolId, + }) + ); + + if (error !== undefined || fetchedRound == null) { + logger.error('Failed to fetch round with applications'); + throw new Error('Failed to fetch round with applications'); + } + + round = fetchedRound; + roundCache[params.alloPoolId] = round; + logger.info( + `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + ); + } + + const application = round.applications.find( + app => app.id === params.alloApplicationId + ); + if (application == null) { + logger.error( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + throw new NotFoundError( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + } + + roundMetadata = round.roundMetadata; + applicationMetadata = application.metadata; } - round = fetchedRound; - roundCache[params.alloPoolId] = round; - logger.info( - `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + const evaluation = await requestEvaluation( + roundMetadata, + applicationMetadata, + evaluationQuestions ); - } - const application = round.applications.find( - app => app.id === params.alloApplicationId + await createEvaluation({ + chainId: params.chainId, + alloPoolId: params.alloPoolId, + alloApplicationId: params.alloApplicationId, + cid: params.cid, + evaluator: params.evaluator, + summaryInput: evaluation, + evaluatorType: EVALUATOR_TYPE.LLM_GPT3, + }); + }); + + await Promise.all(evaluationPromises); + + await new Promise(resolve => setTimeout(resolve, 1000)); + } catch (batchError) { + // Handle any error within the batch (if any promise fails) + logger.error( + 'Error processing batch, skipping to the next one:', + batchError ); - if (application == null) { - logger.error( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - throw new NotFoundError( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - } - - roundMetadata = round.roundMetadata; - applicationMetadata = application.metadata; + // Continue to the next batch even if an error occurred + continue; } - - const evaluation = await requestEvaluation( - roundMetadata, - applicationMetadata, - evaluationQuestions - ); - - await createEvaluation({ - chainId: params.chainId, - alloPoolId: params.alloPoolId, - alloApplicationId: params.alloApplicationId, - cid: params.cid, - evaluator: params.evaluator, - summaryInput: evaluation, - evaluatorType: EVALUATOR_TYPE.LLM_GPT3, - }); - }); - - // Wait for all promises to resolve - await Promise.all(evaluationPromises); + } }; diff --git a/src/controllers/types.ts b/src/controllers/types.ts new file mode 100644 index 0000000..2d37a94 --- /dev/null +++ b/src/controllers/types.ts @@ -0,0 +1,19 @@ +import { type Hex } from 'viem'; + +export interface Signature { + signature: Hex; +} +export interface PoolIdChainId { + alloPoolId: string; + chainId: number; +} + +export interface PoolIdChainIdBody extends PoolIdChainId, Signature {} + +export interface PoolIdChainIdApplicationId extends PoolIdChainId { + alloApplicationId: string; +} + +export interface PoolIdChainIdApplicationIdBody + extends PoolIdChainIdApplicationId, + Signature {} diff --git a/src/entity/Application.ts b/src/entity/Application.ts index c45e896..f484ad1 100644 --- a/src/entity/Application.ts +++ b/src/entity/Application.ts @@ -22,7 +22,9 @@ export class Application { @Column() alloApplicationId: string; - @ManyToOne(() => Pool, pool => pool.applications) + @ManyToOne(() => Pool, pool => pool.applications, { + onDelete: 'CASCADE', + }) pool: Pool; @ManyToOne(() => Profile, profile => profile.applications) diff --git a/src/entity/Evaluation.ts b/src/entity/Evaluation.ts index a642e5c..dac0260 100644 --- a/src/entity/Evaluation.ts +++ b/src/entity/Evaluation.ts @@ -40,12 +40,15 @@ export class Evaluation { @Column() metadataCid: string; - @ManyToOne(() => Application, application => application.evaluations) + @ManyToOne(() => Application, application => application.evaluations, { + onDelete: 'CASCADE', + }) application: Application; @OneToMany( () => EvaluationAnswer, - evaluationAnswer => evaluationAnswer.evaluation + evaluationAnswer => evaluationAnswer.evaluation, + { cascade: true } ) evaluationAnswer: EvaluationAnswer[]; diff --git a/src/entity/EvaluationAnswer.ts b/src/entity/EvaluationAnswer.ts index 6075c41..4f8a919 100644 --- a/src/entity/EvaluationAnswer.ts +++ b/src/entity/EvaluationAnswer.ts @@ -39,12 +39,17 @@ export class EvaluationAnswer { }) answer: AnswerType; - @ManyToOne(() => Evaluation, evaluation => evaluation.evaluationAnswer) + @ManyToOne(() => Evaluation, evaluation => evaluation.evaluationAnswer, { + onDelete: 'CASCADE', + }) evaluation: Evaluation; @ManyToOne( () => EvaluationQuestion, - evaluationQuestion => evaluationQuestion.answers + evaluationQuestion => evaluationQuestion.answers, + { + onDelete: 'CASCADE', + } ) evaluationQuestion: EvaluationQuestion; diff --git a/src/entity/EvaluationQuestion.ts b/src/entity/EvaluationQuestion.ts index 0169fa0..fdef138 100644 --- a/src/entity/EvaluationQuestion.ts +++ b/src/entity/EvaluationQuestion.ts @@ -24,7 +24,9 @@ export class EvaluationQuestion { @OneToMany(() => EvaluationAnswer, answer => answer.evaluationQuestion) answers: EvaluationAnswer[]; - @ManyToOne(() => Pool, pool => pool.questions) + @ManyToOne(() => Pool, pool => pool.questions, { + onDelete: 'CASCADE', + }) pool: Pool; @Column() // Explicitly define the foreign key column for pool diff --git a/src/migration/1731927111769-InitMigration.ts b/src/migration/1732016597076-InitMigration.ts similarity index 83% rename from src/migration/1731927111769-InitMigration.ts rename to src/migration/1732016597076-InitMigration.ts index be2a49b..7bb89a0 100644 --- a/src/migration/1731927111769-InitMigration.ts +++ b/src/migration/1732016597076-InitMigration.ts @@ -1,22 +1,20 @@ import { MigrationInterface, QueryRunner } from "typeorm"; -export class InitMigration1731927111769 implements MigrationInterface { - name = 'InitMigration1731927111769' +export class InitMigration1732016597076 implements MigrationInterface { + name = 'InitMigration1732016597076' public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE TYPE "public"."evaluation_evaluatortype_enum" AS ENUM('human', 'llm_gpt3')`); await queryRunner.query(`CREATE TABLE "evaluation" ("id" SERIAL NOT NULL, "evaluator" character varying(42) NOT NULL, "evaluatorType" "public"."evaluation_evaluatortype_enum" NOT NULL, "summary" character varying NOT NULL, "evaluatorScore" integer NOT NULL, "metadataCid" character varying NOT NULL, "applicationId" integer NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "lastUpdatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_566857ce7db15aa0fb1930b4cdf" UNIQUE ("evaluator", "applicationId"), CONSTRAINT "PK_b72edd439b9db736f55b584fa54" PRIMARY KEY ("id"))`); - await queryRunner.query(`CREATE TYPE "public"."evaluation_answer_answer_enum" AS ENUM('yes', 'no', 'uncertain')`); await queryRunner.query(`CREATE TABLE "evaluation_answer" ("id" SERIAL NOT NULL, "answer" "public"."evaluation_answer_answer_enum" NOT NULL, "evaluationId" integer NOT NULL, "evaluationQuestionId" integer NOT NULL, CONSTRAINT "UQ_5d5571491f885c88023b5f56366" UNIQUE ("evaluationId", "evaluationQuestionId"), CONSTRAINT "PK_26adcf2e8e65214d2558b8f6910" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "evaluation_question" ("id" SERIAL NOT NULL, "questionIndex" integer NOT NULL, "question" character varying NOT NULL, "poolId" integer NOT NULL, CONSTRAINT "UQ_bd9653bd57844a98c0863a0a5b8" UNIQUE ("poolId", "questionIndex"), CONSTRAINT "PK_6ecc0e6614b9c4bc65c6de2c021" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "pool" ("id" SERIAL NOT NULL, "chainId" integer NOT NULL, "alloPoolId" character varying NOT NULL, CONSTRAINT "UQ_72fcaa655b2b7348f4feaf25ea3" UNIQUE ("chainId", "alloPoolId"), CONSTRAINT "PK_db1bfe411e1516c01120b85f8fe" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "application" ("id" SERIAL NOT NULL, "chainId" integer NOT NULL, "alloApplicationId" character varying NOT NULL, "poolId" integer NOT NULL, "profileId" integer, CONSTRAINT "UQ_44dbcb26fba94fd04aaf46392fa" UNIQUE ("alloApplicationId", "poolId", "chainId"), CONSTRAINT "PK_569e0c3e863ebdf5f2408ee1670" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "profile" ("id" SERIAL NOT NULL, "profileId" character varying NOT NULL, CONSTRAINT "UQ_61a193410d652adedb69f7ad680" UNIQUE ("profileId"), CONSTRAINT "PK_3dd8bfc97e4a77c70971591bdcb" PRIMARY KEY ("id"))`); - await queryRunner.query(`ALTER TABLE "evaluation" ADD CONSTRAINT "FK_f459958482585b957ef22cca734" FOREIGN KEY ("applicationId") REFERENCES "application"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_ffe01531544524587279e70fe15" FOREIGN KEY ("evaluationId") REFERENCES "evaluation"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_758462d7b628e9d86fe25861566" FOREIGN KEY ("evaluationQuestionId") REFERENCES "evaluation_question"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "evaluation_question" ADD CONSTRAINT "FK_afa6632818bad5e99f65a6261ed" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_a2d1c7a2c6ee681b42112d41284" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation" ADD CONSTRAINT "FK_f459958482585b957ef22cca734" FOREIGN KEY ("applicationId") REFERENCES "application"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_ffe01531544524587279e70fe15" FOREIGN KEY ("evaluationId") REFERENCES "evaluation"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_758462d7b628e9d86fe25861566" FOREIGN KEY ("evaluationQuestionId") REFERENCES "evaluation_question"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation_question" ADD CONSTRAINT "FK_afa6632818bad5e99f65a6261ed" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_a2d1c7a2c6ee681b42112d41284" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_2537c29f8628eb085b5478e8b00" FOREIGN KEY ("profileId") REFERENCES "profile"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); } @@ -32,9 +30,7 @@ export class InitMigration1731927111769 implements MigrationInterface { await queryRunner.query(`DROP TABLE "pool"`); await queryRunner.query(`DROP TABLE "evaluation_question"`); await queryRunner.query(`DROP TABLE "evaluation_answer"`); - await queryRunner.query(`DROP TYPE "public"."evaluation_answer_answer_enum"`); await queryRunner.query(`DROP TABLE "evaluation"`); - await queryRunner.query(`DROP TYPE "public"."evaluation_evaluatortype_enum"`); } } diff --git a/src/routes/evaluationRoutes.ts b/src/routes/evaluationRoutes.ts index e3916cf..433f3cc 100644 --- a/src/routes/evaluationRoutes.ts +++ b/src/routes/evaluationRoutes.ts @@ -1,5 +1,6 @@ import { evaluateApplication, + recreateEvaluationQuestions, triggerLLMEvaluation, } from '@/controllers/evaluationController'; import { Router } from 'express'; @@ -146,4 +147,50 @@ router.post('/', evaluateApplication); */ router.post('/llm', triggerLLMEvaluation); +/** + * @swagger + * /evaluate/recreate-questions: + * post: + * summary: "Recreate Evaluation Questions" + * description: "This endpoint recreates evaluation questions for a specified pool and chain. **Warning: This will also delete all past evaluations and their associated answers for the specified pool.**" + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * chainId: + * type: integer + * example: 42161 + * alloPoolId: + * type: string + * example: "609" + * signature: + * type: string + * example: "0x1234567890abcdef" + * example: + * chainId: 42161 + * alloPoolId: "609" + * signature: "0xdeadbeef" + * responses: + * 200: + * description: "Evaluation questions recreated successfully" + * content: + * application/json: + * schema: + * type: object + * properties: + * questions: + * type: array + * items: + * type: string + * example: "How would you rate the innovation of the application?" + * 404: + * description: "Failed to fetch round with applications or get evaluation questions" + * 500: + * description: "Internal Server Error" + */ +router.post('/recreate-questions', recreateEvaluationQuestions); + export default router; diff --git a/src/service/EvaluationService.ts b/src/service/EvaluationService.ts index 7315efa..3e4c67d 100644 --- a/src/service/EvaluationService.ts +++ b/src/service/EvaluationService.ts @@ -31,10 +31,38 @@ export interface CreateEvaluationParams { } class EvaluationService { + async cleanEvaluations(): Promise { + const evaluationsWithoutAnswers = await evaluationRepository + .createQueryBuilder('evaluation') + .leftJoinAndSelect('evaluation.evaluationAnswer', 'evaluationAnswer') + .where('evaluationAnswer.id IS NULL') + .getMany(); + + if (evaluationsWithoutAnswers.length > 0) { + await evaluationRepository.remove(evaluationsWithoutAnswers); + } + } + async createEvaluation(evaluation: Partial): Promise { + if (evaluation.evaluator != null && evaluation.applicationId != null) { + await this.deleteExistingEvaluationByEvaluatorAndApplicationId( + evaluation.evaluator, + evaluation.applicationId + ); + } return await evaluationRepository.save(evaluation); } + async deleteExistingEvaluationByEvaluatorAndApplicationId( + evaluator: string, + applicationId: number + ): Promise { + await evaluationRepository.delete({ + evaluator, + applicationId, + }); + } + async createEvaluationWithAnswers({ chainId, alloPoolId,