From 3d6191e655f6ee989b5f1c86d6c26f058f1317e3 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Tue, 19 Nov 2024 15:17:23 +0530 Subject: [PATCH 01/10] wip: re-ealuate --- src/service/EvaluationService.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/service/EvaluationService.ts b/src/service/EvaluationService.ts index 7315efa..b93915d 100644 --- a/src/service/EvaluationService.ts +++ b/src/service/EvaluationService.ts @@ -32,9 +32,27 @@ export interface CreateEvaluationParams { class EvaluationService { async createEvaluation(evaluation: Partial): Promise { + if (evaluation.evaluator != null && evaluation.applicationId != null) { + console.log('=====> DELETING START'); + await this.deleteExistingEvaluationByEvaluatorAndApplicationId( + evaluation.evaluator, + evaluation.applicationId + ); + console.log('=====> DELETING DONE'); + } return await evaluationRepository.save(evaluation); } + async deleteExistingEvaluationByEvaluatorAndApplicationId( + evaluator: string, + applicationId: number + ): Promise { + await evaluationRepository.delete({ + evaluator, + applicationId, + }); + } + async createEvaluationWithAnswers({ chainId, alloPoolId, @@ -69,6 +87,8 @@ class EvaluationService { (1 - totalScore / maxPossibleScore) * 100 ); + console.log('=====> SHIT 1'); + // Create the Evaluation const evaluation = await this.createEvaluation({ evaluator, @@ -80,6 +100,8 @@ class EvaluationService { application, }); + console.log('=====> SHIT 2'); + for (const question of questions) { const evaluationQuestion = await evaluationQuestionService.getEvaluationQuestionsByAlloPoolIdAndQuestionIndex( From af7726abc147265baa8bc5f211a20887dd194431 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Tue, 19 Nov 2024 15:31:20 +0530 Subject: [PATCH 02/10] fix --- src/migration/1731927111769-InitMigration.ts | 4 ++-- src/service/EvaluationService.ts | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/migration/1731927111769-InitMigration.ts b/src/migration/1731927111769-InitMigration.ts index be2a49b..c1276e6 100644 --- a/src/migration/1731927111769-InitMigration.ts +++ b/src/migration/1731927111769-InitMigration.ts @@ -1,4 +1,4 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import { type MigrationInterface, type QueryRunner } from "typeorm"; export class InitMigration1731927111769 implements MigrationInterface { name = 'InitMigration1731927111769' @@ -13,7 +13,7 @@ export class InitMigration1731927111769 implements MigrationInterface { await queryRunner.query(`CREATE TABLE "application" ("id" SERIAL NOT NULL, "chainId" integer NOT NULL, "alloApplicationId" character varying NOT NULL, "poolId" integer NOT NULL, "profileId" integer, CONSTRAINT "UQ_44dbcb26fba94fd04aaf46392fa" UNIQUE ("alloApplicationId", "poolId", "chainId"), CONSTRAINT "PK_569e0c3e863ebdf5f2408ee1670" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "profile" ("id" SERIAL NOT NULL, "profileId" character varying NOT NULL, CONSTRAINT "UQ_61a193410d652adedb69f7ad680" UNIQUE ("profileId"), CONSTRAINT "PK_3dd8bfc97e4a77c70971591bdcb" PRIMARY KEY ("id"))`); await queryRunner.query(`ALTER TABLE "evaluation" ADD CONSTRAINT "FK_f459958482585b957ef22cca734" FOREIGN KEY ("applicationId") REFERENCES "application"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_ffe01531544524587279e70fe15" FOREIGN KEY ("evaluationId") REFERENCES "evaluation"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_ffe01531544524587279e70fe15" FOREIGN KEY ("evaluationId") REFERENCES "evaluation"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_758462d7b628e9d86fe25861566" FOREIGN KEY ("evaluationQuestionId") REFERENCES "evaluation_question"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "evaluation_question" ADD CONSTRAINT "FK_afa6632818bad5e99f65a6261ed" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_a2d1c7a2c6ee681b42112d41284" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); diff --git a/src/service/EvaluationService.ts b/src/service/EvaluationService.ts index b93915d..44c656d 100644 --- a/src/service/EvaluationService.ts +++ b/src/service/EvaluationService.ts @@ -33,12 +33,10 @@ export interface CreateEvaluationParams { class EvaluationService { async createEvaluation(evaluation: Partial): Promise { if (evaluation.evaluator != null && evaluation.applicationId != null) { - console.log('=====> DELETING START'); await this.deleteExistingEvaluationByEvaluatorAndApplicationId( evaluation.evaluator, evaluation.applicationId ); - console.log('=====> DELETING DONE'); } return await evaluationRepository.save(evaluation); } @@ -87,8 +85,6 @@ class EvaluationService { (1 - totalScore / maxPossibleScore) * 100 ); - console.log('=====> SHIT 1'); - // Create the Evaluation const evaluation = await this.createEvaluation({ evaluator, @@ -100,8 +96,6 @@ class EvaluationService { application, }); - console.log('=====> SHIT 2'); - for (const question of questions) { const evaluationQuestion = await evaluationQuestionService.getEvaluationQuestionsByAlloPoolIdAndQuestionIndex( From ca2cceb18694c441f009675d35a6b017a87fc138 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Tue, 19 Nov 2024 15:34:27 +0530 Subject: [PATCH 03/10] f --- src/entity/Evaluation.ts | 3 ++- src/entity/EvaluationAnswer.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/entity/Evaluation.ts b/src/entity/Evaluation.ts index a642e5c..71754fb 100644 --- a/src/entity/Evaluation.ts +++ b/src/entity/Evaluation.ts @@ -45,7 +45,8 @@ export class Evaluation { @OneToMany( () => EvaluationAnswer, - evaluationAnswer => evaluationAnswer.evaluation + evaluationAnswer => evaluationAnswer.evaluation, + { cascade: true } ) evaluationAnswer: EvaluationAnswer[]; diff --git a/src/entity/EvaluationAnswer.ts b/src/entity/EvaluationAnswer.ts index 6075c41..ad82fcb 100644 --- a/src/entity/EvaluationAnswer.ts +++ b/src/entity/EvaluationAnswer.ts @@ -39,7 +39,9 @@ export class EvaluationAnswer { }) answer: AnswerType; - @ManyToOne(() => Evaluation, evaluation => evaluation.evaluationAnswer) + @ManyToOne(() => Evaluation, evaluation => evaluation.evaluationAnswer, { + onDelete: 'CASCADE', + }) evaluation: Evaluation; @ManyToOne( From 354f33515c002fb30f4344b61d3e41d2325703da Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Tue, 19 Nov 2024 11:16:30 +0100 Subject: [PATCH 04/10] geerate migration file --- src/entity/Evaluation.ts | 3 +-- ...769-InitMigration.ts => 1732011017611-migration.ts} | 10 +++------- 2 files changed, 4 insertions(+), 9 deletions(-) rename src/migration/{1731927111769-InitMigration.ts => 1732011017611-migration.ts} (88%) diff --git a/src/entity/Evaluation.ts b/src/entity/Evaluation.ts index 71754fb..a642e5c 100644 --- a/src/entity/Evaluation.ts +++ b/src/entity/Evaluation.ts @@ -45,8 +45,7 @@ export class Evaluation { @OneToMany( () => EvaluationAnswer, - evaluationAnswer => evaluationAnswer.evaluation, - { cascade: true } + evaluationAnswer => evaluationAnswer.evaluation ) evaluationAnswer: EvaluationAnswer[]; diff --git a/src/migration/1731927111769-InitMigration.ts b/src/migration/1732011017611-migration.ts similarity index 88% rename from src/migration/1731927111769-InitMigration.ts rename to src/migration/1732011017611-migration.ts index c1276e6..12e3e80 100644 --- a/src/migration/1731927111769-InitMigration.ts +++ b/src/migration/1732011017611-migration.ts @@ -1,12 +1,10 @@ -import { type MigrationInterface, type QueryRunner } from "typeorm"; +import { MigrationInterface, QueryRunner } from "typeorm"; -export class InitMigration1731927111769 implements MigrationInterface { - name = 'InitMigration1731927111769' +export class Migration1732011017611 implements MigrationInterface { + name = 'Migration1732011017611' public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE TYPE "public"."evaluation_evaluatortype_enum" AS ENUM('human', 'llm_gpt3')`); await queryRunner.query(`CREATE TABLE "evaluation" ("id" SERIAL NOT NULL, "evaluator" character varying(42) NOT NULL, "evaluatorType" "public"."evaluation_evaluatortype_enum" NOT NULL, "summary" character varying NOT NULL, "evaluatorScore" integer NOT NULL, "metadataCid" character varying NOT NULL, "applicationId" integer NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "lastUpdatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_566857ce7db15aa0fb1930b4cdf" UNIQUE ("evaluator", "applicationId"), CONSTRAINT "PK_b72edd439b9db736f55b584fa54" PRIMARY KEY ("id"))`); - await queryRunner.query(`CREATE TYPE "public"."evaluation_answer_answer_enum" AS ENUM('yes', 'no', 'uncertain')`); await queryRunner.query(`CREATE TABLE "evaluation_answer" ("id" SERIAL NOT NULL, "answer" "public"."evaluation_answer_answer_enum" NOT NULL, "evaluationId" integer NOT NULL, "evaluationQuestionId" integer NOT NULL, CONSTRAINT "UQ_5d5571491f885c88023b5f56366" UNIQUE ("evaluationId", "evaluationQuestionId"), CONSTRAINT "PK_26adcf2e8e65214d2558b8f6910" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "evaluation_question" ("id" SERIAL NOT NULL, "questionIndex" integer NOT NULL, "question" character varying NOT NULL, "poolId" integer NOT NULL, CONSTRAINT "UQ_bd9653bd57844a98c0863a0a5b8" UNIQUE ("poolId", "questionIndex"), CONSTRAINT "PK_6ecc0e6614b9c4bc65c6de2c021" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "pool" ("id" SERIAL NOT NULL, "chainId" integer NOT NULL, "alloPoolId" character varying NOT NULL, CONSTRAINT "UQ_72fcaa655b2b7348f4feaf25ea3" UNIQUE ("chainId", "alloPoolId"), CONSTRAINT "PK_db1bfe411e1516c01120b85f8fe" PRIMARY KEY ("id"))`); @@ -32,9 +30,7 @@ export class InitMigration1731927111769 implements MigrationInterface { await queryRunner.query(`DROP TABLE "pool"`); await queryRunner.query(`DROP TABLE "evaluation_question"`); await queryRunner.query(`DROP TABLE "evaluation_answer"`); - await queryRunner.query(`DROP TYPE "public"."evaluation_answer_answer_enum"`); await queryRunner.query(`DROP TABLE "evaluation"`); - await queryRunner.query(`DROP TYPE "public"."evaluation_evaluatortype_enum"`); } } From 85dd2c75285c78f736dc139d0e2e13b5e3a12bb0 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Tue, 19 Nov 2024 11:55:19 +0100 Subject: [PATCH 05/10] updated cascde --- src/controllers/evaluationController.ts | 14 ++--- src/controllers/poolController.ts | 51 ++++++++++++++++++- src/controllers/types.ts | 12 +++++ src/entity/Application.ts | 4 +- src/entity/Evaluation.ts | 4 +- src/entity/EvaluationAnswer.ts | 5 +- src/entity/EvaluationQuestion.ts | 4 +- ...tion.ts => 1732012141411-InitMigration.ts} | 12 ++--- 8 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 src/controllers/types.ts rename src/migration/{1732011017611-migration.ts => 1732012141411-InitMigration.ts} (92%) diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index f8046f7..3571e43 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -24,6 +24,10 @@ import { import { type Evaluation, EVALUATOR_TYPE } from '@/entity/Evaluation'; import { IsNullError, NotFoundError } from '@/errors'; import { type Hex } from 'viem'; +import type { + PoolIdChainIdApplicationId, + PoolIdChainIdApplicationIdBody, +} from './types'; const logger = createLogger(); @@ -151,16 +155,6 @@ export interface CreateLLMEvaluationParams { applicationMetadata?: ApplicationMetadata; questions?: PromptEvaluationQuestions; } -interface PoolIdChainIdApplicationId { - alloPoolId: string; - chainId: number; - alloApplicationId: string; -} - -interface PoolIdChainIdApplicationIdBody extends PoolIdChainIdApplicationId { - signature: Hex; -} - export const triggerLLMEvaluation = async ( req: Request, res: Response diff --git a/src/controllers/poolController.ts b/src/controllers/poolController.ts index 66faf38..51f3870 100644 --- a/src/controllers/poolController.ts +++ b/src/controllers/poolController.ts @@ -1,6 +1,11 @@ import type { Request, Response } from 'express'; import poolService from '@/service/PoolService'; -import { addressFrom, catchError, validateRequest } from '@/utils'; +import { + addressFrom, + catchError, + isPoolManager, + validateRequest, +} from '@/utils'; import { createLogger } from '@/logger'; import { indexerClient, @@ -20,6 +25,10 @@ import { import { type Pool } from '@/entity/Pool'; import { IsNullError, NotFoundError } from '@/errors'; import { env } from '@/env'; +import type { + PoolIdChainIdApplicationId, + PoolIdChainIdApplicationIdBody, +} from './types'; const logger = createLogger(); @@ -113,6 +122,46 @@ export const syncPool = async (req: Request, res: Response): Promise => { res.status(200).json({ message: 'pool synced successfully' }); }; +export const recreateEvaluationQuestions = async ( + req: Request, + res: Response +): Promise => { + validateRequest(req, res); + + const { alloPoolId, chainId, alloApplicationId, signature } = + req.body as PoolIdChainIdApplicationIdBody; + + const isAllowed = await isPoolManager( + { alloPoolId, chainId, alloApplicationId }, + signature, + chainId, + alloPoolId + ); + + if (!isAllowed) { + logger.warn( + `User with address: ${signature} is not allowed to evaluate application` + ); + res.status(403).json({ message: 'Unauthorized' }); + return; + } + + // ---- LLM evaluation questions ---- + // Fetch evaluation questions from the pool or request new questions + const [evalQuestionsError, evaluationQuestions] = await catchError( + handlePoolEvaluationQuestions(pool, indexerPoolData.roundMetadata) + ); + + // Handle errors during the evaluation question handling + if (evalQuestionsError != null || evaluationQuestions == null) { + res.status(500).json({ + message: 'Error handling evaluation questions', + error: evalQuestionsError?.message, + }); + throw new IsNullError(`Error handling evaluation questions`); + } +}; + const handlePoolEvaluationQuestions = async ( pool: Pool, poolMetadata: IndexerRoundMetadata diff --git a/src/controllers/types.ts b/src/controllers/types.ts new file mode 100644 index 0000000..e45f4ff --- /dev/null +++ b/src/controllers/types.ts @@ -0,0 +1,12 @@ +import { type Hex } from 'viem'; + +export interface PoolIdChainIdApplicationId { + alloPoolId: string; + chainId: number; + alloApplicationId: string; +} + +export interface PoolIdChainIdApplicationIdBody + extends PoolIdChainIdApplicationId { + signature: Hex; +} diff --git a/src/entity/Application.ts b/src/entity/Application.ts index c45e896..f484ad1 100644 --- a/src/entity/Application.ts +++ b/src/entity/Application.ts @@ -22,7 +22,9 @@ export class Application { @Column() alloApplicationId: string; - @ManyToOne(() => Pool, pool => pool.applications) + @ManyToOne(() => Pool, pool => pool.applications, { + onDelete: 'CASCADE', + }) pool: Pool; @ManyToOne(() => Profile, profile => profile.applications) diff --git a/src/entity/Evaluation.ts b/src/entity/Evaluation.ts index a642e5c..3859bc2 100644 --- a/src/entity/Evaluation.ts +++ b/src/entity/Evaluation.ts @@ -40,7 +40,9 @@ export class Evaluation { @Column() metadataCid: string; - @ManyToOne(() => Application, application => application.evaluations) + @ManyToOne(() => Application, application => application.evaluations, { + onDelete: 'CASCADE', + }) application: Application; @OneToMany( diff --git a/src/entity/EvaluationAnswer.ts b/src/entity/EvaluationAnswer.ts index ad82fcb..4f8a919 100644 --- a/src/entity/EvaluationAnswer.ts +++ b/src/entity/EvaluationAnswer.ts @@ -46,7 +46,10 @@ export class EvaluationAnswer { @ManyToOne( () => EvaluationQuestion, - evaluationQuestion => evaluationQuestion.answers + evaluationQuestion => evaluationQuestion.answers, + { + onDelete: 'CASCADE', + } ) evaluationQuestion: EvaluationQuestion; diff --git a/src/entity/EvaluationQuestion.ts b/src/entity/EvaluationQuestion.ts index 0169fa0..fdef138 100644 --- a/src/entity/EvaluationQuestion.ts +++ b/src/entity/EvaluationQuestion.ts @@ -24,7 +24,9 @@ export class EvaluationQuestion { @OneToMany(() => EvaluationAnswer, answer => answer.evaluationQuestion) answers: EvaluationAnswer[]; - @ManyToOne(() => Pool, pool => pool.questions) + @ManyToOne(() => Pool, pool => pool.questions, { + onDelete: 'CASCADE', + }) pool: Pool; @Column() // Explicitly define the foreign key column for pool diff --git a/src/migration/1732011017611-migration.ts b/src/migration/1732012141411-InitMigration.ts similarity index 92% rename from src/migration/1732011017611-migration.ts rename to src/migration/1732012141411-InitMigration.ts index 12e3e80..2630784 100644 --- a/src/migration/1732011017611-migration.ts +++ b/src/migration/1732012141411-InitMigration.ts @@ -1,7 +1,7 @@ import { MigrationInterface, QueryRunner } from "typeorm"; -export class Migration1732011017611 implements MigrationInterface { - name = 'Migration1732011017611' +export class InitMigration1732012141411 implements MigrationInterface { + name = 'InitMigration1732012141411' public async up(queryRunner: QueryRunner): Promise { await queryRunner.query(`CREATE TABLE "evaluation" ("id" SERIAL NOT NULL, "evaluator" character varying(42) NOT NULL, "evaluatorType" "public"."evaluation_evaluatortype_enum" NOT NULL, "summary" character varying NOT NULL, "evaluatorScore" integer NOT NULL, "metadataCid" character varying NOT NULL, "applicationId" integer NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "lastUpdatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_566857ce7db15aa0fb1930b4cdf" UNIQUE ("evaluator", "applicationId"), CONSTRAINT "PK_b72edd439b9db736f55b584fa54" PRIMARY KEY ("id"))`); @@ -10,11 +10,11 @@ export class Migration1732011017611 implements MigrationInterface { await queryRunner.query(`CREATE TABLE "pool" ("id" SERIAL NOT NULL, "chainId" integer NOT NULL, "alloPoolId" character varying NOT NULL, CONSTRAINT "UQ_72fcaa655b2b7348f4feaf25ea3" UNIQUE ("chainId", "alloPoolId"), CONSTRAINT "PK_db1bfe411e1516c01120b85f8fe" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "application" ("id" SERIAL NOT NULL, "chainId" integer NOT NULL, "alloApplicationId" character varying NOT NULL, "poolId" integer NOT NULL, "profileId" integer, CONSTRAINT "UQ_44dbcb26fba94fd04aaf46392fa" UNIQUE ("alloApplicationId", "poolId", "chainId"), CONSTRAINT "PK_569e0c3e863ebdf5f2408ee1670" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "profile" ("id" SERIAL NOT NULL, "profileId" character varying NOT NULL, CONSTRAINT "UQ_61a193410d652adedb69f7ad680" UNIQUE ("profileId"), CONSTRAINT "PK_3dd8bfc97e4a77c70971591bdcb" PRIMARY KEY ("id"))`); - await queryRunner.query(`ALTER TABLE "evaluation" ADD CONSTRAINT "FK_f459958482585b957ef22cca734" FOREIGN KEY ("applicationId") REFERENCES "application"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation" ADD CONSTRAINT "FK_f459958482585b957ef22cca734" FOREIGN KEY ("applicationId") REFERENCES "application"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_ffe01531544524587279e70fe15" FOREIGN KEY ("evaluationId") REFERENCES "evaluation"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_758462d7b628e9d86fe25861566" FOREIGN KEY ("evaluationQuestionId") REFERENCES "evaluation_question"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "evaluation_question" ADD CONSTRAINT "FK_afa6632818bad5e99f65a6261ed" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_a2d1c7a2c6ee681b42112d41284" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation_answer" ADD CONSTRAINT "FK_758462d7b628e9d86fe25861566" FOREIGN KEY ("evaluationQuestionId") REFERENCES "evaluation_question"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "evaluation_question" ADD CONSTRAINT "FK_afa6632818bad5e99f65a6261ed" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_a2d1c7a2c6ee681b42112d41284" FOREIGN KEY ("poolId") REFERENCES "pool"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "application" ADD CONSTRAINT "FK_2537c29f8628eb085b5478e8b00" FOREIGN KEY ("profileId") REFERENCES "profile"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); } From ac812d609f69350f1dad572bb78bd59675eef2e5 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Tue, 19 Nov 2024 12:00:47 +0100 Subject: [PATCH 06/10] fix poolController --- src/controllers/poolController.ts | 51 +------------------------------ 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/src/controllers/poolController.ts b/src/controllers/poolController.ts index 51f3870..66faf38 100644 --- a/src/controllers/poolController.ts +++ b/src/controllers/poolController.ts @@ -1,11 +1,6 @@ import type { Request, Response } from 'express'; import poolService from '@/service/PoolService'; -import { - addressFrom, - catchError, - isPoolManager, - validateRequest, -} from '@/utils'; +import { addressFrom, catchError, validateRequest } from '@/utils'; import { createLogger } from '@/logger'; import { indexerClient, @@ -25,10 +20,6 @@ import { import { type Pool } from '@/entity/Pool'; import { IsNullError, NotFoundError } from '@/errors'; import { env } from '@/env'; -import type { - PoolIdChainIdApplicationId, - PoolIdChainIdApplicationIdBody, -} from './types'; const logger = createLogger(); @@ -122,46 +113,6 @@ export const syncPool = async (req: Request, res: Response): Promise => { res.status(200).json({ message: 'pool synced successfully' }); }; -export const recreateEvaluationQuestions = async ( - req: Request, - res: Response -): Promise => { - validateRequest(req, res); - - const { alloPoolId, chainId, alloApplicationId, signature } = - req.body as PoolIdChainIdApplicationIdBody; - - const isAllowed = await isPoolManager( - { alloPoolId, chainId, alloApplicationId }, - signature, - chainId, - alloPoolId - ); - - if (!isAllowed) { - logger.warn( - `User with address: ${signature} is not allowed to evaluate application` - ); - res.status(403).json({ message: 'Unauthorized' }); - return; - } - - // ---- LLM evaluation questions ---- - // Fetch evaluation questions from the pool or request new questions - const [evalQuestionsError, evaluationQuestions] = await catchError( - handlePoolEvaluationQuestions(pool, indexerPoolData.roundMetadata) - ); - - // Handle errors during the evaluation question handling - if (evalQuestionsError != null || evaluationQuestions == null) { - res.status(500).json({ - message: 'Error handling evaluation questions', - error: evalQuestionsError?.message, - }); - throw new IsNullError(`Error handling evaluation questions`); - } -}; - const handlePoolEvaluationQuestions = async ( pool: Pool, poolMetadata: IndexerRoundMetadata From 1e042872c3e7888bc281ccb631acf810c1e46a8f Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Tue, 19 Nov 2024 12:52:19 +0100 Subject: [PATCH 07/10] re-create evaluation questions --- src/controllers/evaluationController.ts | 65 +++++++++++++++++++ src/controllers/types.ts | 15 +++-- src/entity/Evaluation.ts | 3 +- ...tion.ts => 1732016597076-InitMigration.ts} | 4 +- src/routes/evaluationRoutes.ts | 49 +++++++++++++- src/service/EvaluationService.ts | 12 ++++ 6 files changed, 140 insertions(+), 8 deletions(-) rename src/migration/{1732012141411-InitMigration.ts => 1732016597076-InitMigration.ts} (97%) diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index 3571e43..e305186 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -14,6 +14,7 @@ import poolService from '@/service/PoolService'; import { type PromptEvaluationQuestions, requestEvaluation, + requestEvaluationQuestions, } from '@/ext/openai'; import { type ApplicationMetadata, @@ -25,9 +26,12 @@ import { type Evaluation, EVALUATOR_TYPE } from '@/entity/Evaluation'; import { IsNullError, NotFoundError } from '@/errors'; import { type Hex } from 'viem'; import type { + PoolIdChainId, PoolIdChainIdApplicationId, PoolIdChainIdApplicationIdBody, + PoolIdChainIdBody, } from './types'; +import evaluationQuestionService from '@/service/EvaluationQuestionService'; const logger = createLogger(); @@ -35,6 +39,67 @@ interface EvaluationBody extends CreateEvaluationParams { signature: Hex; } +export const recreateEvaluationQuestions = async ( + req: Request, + res: Response +): Promise => { + const { chainId, alloPoolId, signature } = req.body as PoolIdChainIdBody; + + const isAllowed = await isPoolManager( + { alloPoolId, chainId }, + signature, + chainId, + alloPoolId + ); + + if (!isAllowed) { + logger.warn( + `User with address: ${signature} is not allowed to evaluate application` + ); + res.status(403).json({ message: 'Unauthorized' }); + return; + } + + const [error, roundWithApplications] = await catchError( + indexerClient.getRoundWithApplications({ + chainId, + roundId: alloPoolId, + }) + ); + + if ( + error !== undefined || + roundWithApplications === undefined || + roundWithApplications?.roundMetadata === undefined + ) { + logger.error('Failed to fetch round with applications'); + res + .status(404) + .json({ message: 'Failed to fetch round with applications' }); + return; + } + + const evaluationQuestions = await requestEvaluationQuestions( + roundWithApplications.roundMetadata + ); + + if (evaluationQuestions === null || evaluationQuestions.length === 0) { + logger.error('Failed to get evaluation questions'); + res.status(404).json({ message: 'Failed to get evaluation questions' }); + return; + } + + await evaluationQuestionService.resetEvaluationQuestions( + chainId, + alloPoolId, + evaluationQuestions + ); + + await evaluationService.cleanEvaluations(); + + res.status(200).json(evaluationQuestions); +}; + export const evaluateApplication = async ( req: Request, res: Response diff --git a/src/controllers/types.ts b/src/controllers/types.ts index e45f4ff..2d37a94 100644 --- a/src/controllers/types.ts +++ b/src/controllers/types.ts @@ -1,12 +1,19 @@ import { type Hex } from 'viem'; -export interface PoolIdChainIdApplicationId { +export interface Signature { + signature: Hex; +} +export interface PoolIdChainId { alloPoolId: string; chainId: number; +} + +export interface PoolIdChainIdBody extends PoolIdChainId, Signature {} + +export interface PoolIdChainIdApplicationId extends PoolIdChainId { alloApplicationId: string; } export interface PoolIdChainIdApplicationIdBody - extends PoolIdChainIdApplicationId { - signature: Hex; -} + extends PoolIdChainIdApplicationId, + Signature {} diff --git a/src/entity/Evaluation.ts b/src/entity/Evaluation.ts index 3859bc2..dac0260 100644 --- a/src/entity/Evaluation.ts +++ b/src/entity/Evaluation.ts @@ -47,7 +47,8 @@ export class Evaluation { @OneToMany( () => EvaluationAnswer, - evaluationAnswer => evaluationAnswer.evaluation + evaluationAnswer => evaluationAnswer.evaluation, + { cascade: true } ) evaluationAnswer: EvaluationAnswer[]; diff --git a/src/migration/1732012141411-InitMigration.ts b/src/migration/1732016597076-InitMigration.ts similarity index 97% rename from src/migration/1732012141411-InitMigration.ts rename to src/migration/1732016597076-InitMigration.ts index 2630784..7bb89a0 100644 --- a/src/migration/1732012141411-InitMigration.ts +++ b/src/migration/1732016597076-InitMigration.ts @@ -1,7 +1,7 @@ import { MigrationInterface, QueryRunner } from "typeorm"; -export class InitMigration1732012141411 implements MigrationInterface { - name = 'InitMigration1732012141411' +export class InitMigration1732016597076 implements MigrationInterface { + name = 'InitMigration1732016597076' public async up(queryRunner: QueryRunner): Promise { await queryRunner.query(`CREATE TABLE "evaluation" ("id" SERIAL NOT NULL, "evaluator" character varying(42) NOT NULL, "evaluatorType" "public"."evaluation_evaluatortype_enum" NOT NULL, "summary" character varying NOT NULL, "evaluatorScore" integer NOT NULL, "metadataCid" character varying NOT NULL, "applicationId" integer NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "lastUpdatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_566857ce7db15aa0fb1930b4cdf" UNIQUE ("evaluator", "applicationId"), CONSTRAINT "PK_b72edd439b9db736f55b584fa54" PRIMARY KEY ("id"))`); diff --git a/src/routes/evaluationRoutes.ts b/src/routes/evaluationRoutes.ts index e3916cf..65d1b1c 100644 --- a/src/routes/evaluationRoutes.ts +++ b/src/routes/evaluationRoutes.ts @@ -1,5 +1,6 @@ import { evaluateApplication, + recreateEvaluationQuestions, triggerLLMEvaluation, } from '@/controllers/evaluationController'; import { Router } from 'express'; @@ -146,4 +147,50 @@ router.post('/', evaluateApplication); */ router.post('/llm', triggerLLMEvaluation); -export default router; +/** + * @swagger + * /evaluate/recreate-questions: + * post: + * summary: "Recreate Evaluation Questions" + * description: "This endpoint recreates evaluation questions for a specified pool and chain." + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * chainId: + * type: integer + * example: 42161 + * alloPoolId: + * type: string + * example: "609" + * signature: + * type: string + * example: "0x1234567890abcdef" + * example: + * chainId: 42161 + * alloPoolId: "609" + * signature: "0xdeadbeef" + * responses: + * 200: + * description: "Evaluation questions recreated successfully" + * content: + * application/json: + * schema: + * type: object + * properties: + * questions: + * type: array + * items: + * type: string + * example: "How would you rate the innovation of the application?" + * 404: + * description: "Failed to fetch round with applications or get evaluation questions" + * 500: + * description: "Internal Server Error" + */ +router.post('/recreate-questions', recreateEvaluationQuestions); + +export default router; \ No newline at end of file diff --git a/src/service/EvaluationService.ts b/src/service/EvaluationService.ts index 44c656d..3e4c67d 100644 --- a/src/service/EvaluationService.ts +++ b/src/service/EvaluationService.ts @@ -31,6 +31,18 @@ export interface CreateEvaluationParams { } class EvaluationService { + async cleanEvaluations(): Promise { + const evaluationsWithoutAnswers = await evaluationRepository + .createQueryBuilder('evaluation') + .leftJoinAndSelect('evaluation.evaluationAnswer', 'evaluationAnswer') + .where('evaluationAnswer.id IS NULL') + .getMany(); + + if (evaluationsWithoutAnswers.length > 0) { + await evaluationRepository.remove(evaluationsWithoutAnswers); + } + } + async createEvaluation(evaluation: Partial): Promise { if (evaluation.evaluator != null && evaluation.applicationId != null) { await this.deleteExistingEvaluationByEvaluatorAndApplicationId( From b61f8ed9d6c08d147e4782f9b0522c8f7a991687 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Tue, 19 Nov 2024 12:54:31 +0100 Subject: [PATCH 08/10] update comment --- src/routes/evaluationRoutes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/evaluationRoutes.ts b/src/routes/evaluationRoutes.ts index 65d1b1c..0d37264 100644 --- a/src/routes/evaluationRoutes.ts +++ b/src/routes/evaluationRoutes.ts @@ -152,7 +152,7 @@ router.post('/llm', triggerLLMEvaluation); * /evaluate/recreate-questions: * post: * summary: "Recreate Evaluation Questions" - * description: "This endpoint recreates evaluation questions for a specified pool and chain." + * description: "This endpoint recreates evaluation questions for a specified pool and chain. **Warning: This will also delete all past evaluations and their associated answers for the specified pool.**" * requestBody: * required: true * content: From 14eb0af4e8440b70b55dafb61b1f747a99061672 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Tue, 19 Nov 2024 13:42:20 +0100 Subject: [PATCH 09/10] batch open ai --- src/controllers/evaluationController.ts | 178 ++++++++++++++---------- 1 file changed, 103 insertions(+), 75 deletions(-) diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index e305186..19b32fd 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -288,91 +288,119 @@ export const triggerLLMEvaluation = async ( } }; +const batchPromises = (array: T[], batchSize: number): T[][] => { + const batches: T[][] = []; + for (let i = 0; i < array.length; i += batchSize) { + batches.push(array.slice(i, i + batchSize)); + } + return batches; +}; + export const createLLMEvaluations = async ( paramsArray: CreateLLMEvaluationParams[] ): Promise => { const roundCache: Record = {}; - const evaluationPromises = paramsArray.map(async params => { - const evaluationQuestions = - params.questions === undefined || params.questions.length === 0 - ? await evaluationService.getQuestionsByChainAndAlloPoolId( - params.chainId, - params.alloPoolId - ) - : params.questions; - - if (evaluationQuestions === null || evaluationQuestions.length === 0) { - logger.error('createLLMEvaluations:Failed to get evaluation questions'); - throw new Error('Failed to get evaluation questions'); - } - - let roundMetadata = params.roundMetadata; - let applicationMetadata = params.applicationMetadata; - // Check if the round is already in cache - if (roundMetadata == null || applicationMetadata == null) { - let round: RoundWithApplications | null; - - // If the round is cached, use it - if (roundCache[params.alloPoolId] != null) { - round = roundCache[params.alloPoolId]; - logger.debug( - `Using cached round data for roundId: ${params.alloPoolId}` - ); - } else { - // Fetch the round and store it in the cache - const [error, fetchedRound] = await catchError( - indexerClient.getRoundWithApplications({ - chainId: params.chainId, - roundId: params.alloPoolId, - }) - ); + // Split the paramsArray into batches of 10 + const batchedParams = batchPromises(paramsArray, 10); + + for (const batch of batchedParams) { + try { + // Process each batch of promises concurrently + const evaluationPromises = batch.map(async params => { + const evaluationQuestions = + params.questions === undefined || params.questions.length === 0 + ? await evaluationService.getQuestionsByChainAndAlloPoolId( + params.chainId, + params.alloPoolId + ) + : params.questions; + + if (evaluationQuestions === null || evaluationQuestions.length === 0) { + logger.error( + 'createLLMEvaluations:Failed to get evaluation questions' + ); + throw new Error('Failed to get evaluation questions'); + } - if (error !== undefined || fetchedRound == null) { - logger.error('Failed to fetch round with applications'); - throw new Error('Failed to fetch round with applications'); + let roundMetadata = params.roundMetadata; + let applicationMetadata = params.applicationMetadata; + + // Check if the round is already in cache + if (roundMetadata == null || applicationMetadata == null) { + let round: RoundWithApplications | null; + + // If the round is cached, use it + if (roundCache[params.alloPoolId] != null) { + round = roundCache[params.alloPoolId]; + logger.debug( + `Using cached round data for roundId: ${params.alloPoolId}` + ); + } else { + // Fetch the round and store it in the cache + const [error, fetchedRound] = await catchError( + indexerClient.getRoundWithApplications({ + chainId: params.chainId, + roundId: params.alloPoolId, + }) + ); + + if (error !== undefined || fetchedRound == null) { + logger.error('Failed to fetch round with applications'); + throw new Error('Failed to fetch round with applications'); + } + + round = fetchedRound; + roundCache[params.alloPoolId] = round; + logger.info( + `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + ); + } + + const application = round.applications.find( + app => app.id === params.alloApplicationId + ); + if (application == null) { + logger.error( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + throw new NotFoundError( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + } + + roundMetadata = round.roundMetadata; + applicationMetadata = application.metadata; } - round = fetchedRound; - roundCache[params.alloPoolId] = round; - logger.info( - `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + const evaluation = await requestEvaluation( + roundMetadata, + applicationMetadata, + evaluationQuestions ); - } - const application = round.applications.find( - app => app.id === params.alloApplicationId + await createEvaluation({ + chainId: params.chainId, + alloPoolId: params.alloPoolId, + alloApplicationId: params.alloApplicationId, + cid: params.cid, + evaluator: params.evaluator, + summaryInput: evaluation, + evaluatorType: EVALUATOR_TYPE.LLM_GPT3, + }); + }); + + await Promise.all(evaluationPromises); + + await new Promise(resolve => setTimeout(resolve, 1000)); + } catch (batchError) { + // Handle any error within the batch (if any promise fails) + logger.error( + 'Error processing batch, skipping to the next one:', + batchError ); - if (application == null) { - logger.error( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - throw new NotFoundError( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - } - - roundMetadata = round.roundMetadata; - applicationMetadata = application.metadata; + // Continue to the next batch even if an error occurred + continue; } - - const evaluation = await requestEvaluation( - roundMetadata, - applicationMetadata, - evaluationQuestions - ); - - await createEvaluation({ - chainId: params.chainId, - alloPoolId: params.alloPoolId, - alloApplicationId: params.alloApplicationId, - cid: params.cid, - evaluator: params.evaluator, - summaryInput: evaluation, - evaluatorType: EVALUATOR_TYPE.LLM_GPT3, - }); - }); - - // Wait for all promises to resolve - await Promise.all(evaluationPromises); + } }; From a5f505d7b01118cae9ba8dc899dc17cfce3a10f5 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Tue, 19 Nov 2024 18:16:55 +0530 Subject: [PATCH 10/10] fix ci --- src/routes/evaluationRoutes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/evaluationRoutes.ts b/src/routes/evaluationRoutes.ts index 0d37264..433f3cc 100644 --- a/src/routes/evaluationRoutes.ts +++ b/src/routes/evaluationRoutes.ts @@ -193,4 +193,4 @@ router.post('/llm', triggerLLMEvaluation); */ router.post('/recreate-questions', recreateEvaluationQuestions); -export default router; \ No newline at end of file +export default router;