From 5192ecea30b7e5ed370c834010e31fb987c37793 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Thu, 5 Dec 2024 13:46:01 +0100 Subject: [PATCH 1/4] update route and prompt --- src/controllers/evaluationController.ts | 156 +++++++++++++----------- src/controllers/poolController.ts | 32 +++-- src/ext/openai/prompt.ts | 8 +- src/routes/poolRoutes.ts | 36 ++++-- 4 files changed, 138 insertions(+), 94 deletions(-) diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index 8092967..d2f00c2 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -300,8 +300,9 @@ const batchPromises = (array: T[], batchSize: number): T[][] => { export const createLLMEvaluations = async ( paramsArray: CreateLLMEvaluationParams[] -): Promise => { +): Promise => { const roundCache: Record = {}; + const failedProjects: string[] = []; // Split the paramsArray into batches of 10 const batchedParams = batchPromises(paramsArray, 10); @@ -310,86 +311,95 @@ export const createLLMEvaluations = async ( try { // Process each batch of promises concurrently const evaluationPromises = batch.map(async params => { - const evaluationQuestions = - params.questions === undefined || params.questions.length === 0 - ? await evaluationService.getQuestionsByChainAndAlloPoolId( - params.chainId, - params.alloPoolId - ) - : params.questions; - - if (evaluationQuestions === null || evaluationQuestions.length === 0) { - logger.error( - 'createLLMEvaluations:Failed to get evaluation questions' - ); - throw new Error('Failed to get evaluation questions'); - } - - let roundMetadata = params.roundMetadata; - let applicationMetadata = params.applicationMetadata; - - // Check if the round is already in cache - if (roundMetadata == null || applicationMetadata == null) { - let round: RoundWithApplications | null; - - // If the round is cached, use it - if (roundCache[params.alloPoolId] != null) { - round = roundCache[params.alloPoolId]; - logger.debug( - `Using cached round data for roundId: ${params.alloPoolId}` - ); - } else { - // Fetch the round and store it in the cache - const [error, fetchedRound] = await catchError( - indexerClient.getRoundWithApplications({ - chainId: params.chainId, - roundId: params.alloPoolId, - }) + try { + const evaluationQuestions = + params.questions === undefined || params.questions.length === 0 + ? await evaluationService.getQuestionsByChainAndAlloPoolId( + params.chainId, + params.alloPoolId + ) + : params.questions; + + if ( + evaluationQuestions === null || + evaluationQuestions.length === 0 + ) { + logger.error( + 'createLLMEvaluations:Failed to get evaluation questions' ); + throw new Error('Failed to get evaluation questions'); + } - if (error !== undefined || fetchedRound == null) { - logger.error('Failed to fetch round with applications'); - throw new Error('Failed to fetch round with applications'); + let roundMetadata = params.roundMetadata; + let applicationMetadata = params.applicationMetadata; + + // Check if the round is already in cache + if (roundMetadata == null || applicationMetadata == null) { + let round: RoundWithApplications | null; + + // If the round is cached, use it + if (roundCache[params.alloPoolId] != null) { + round = roundCache[params.alloPoolId]; + logger.debug( + `Using cached round data for roundId: ${params.alloPoolId}` + ); + } else { + // Fetch the round and store it in the cache + const [error, fetchedRound] = await catchError( + indexerClient.getRoundWithApplications({ + chainId: params.chainId, + roundId: params.alloPoolId, + }) + ); + + if (error !== undefined || fetchedRound == null) { + logger.error('Failed to fetch round with applications'); + throw new Error('Failed to fetch round with applications'); + } + + round = fetchedRound; + roundCache[params.alloPoolId] = round; + logger.info( + `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + ); } - round = fetchedRound; - roundCache[params.alloPoolId] = round; - logger.info( - `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications` + const application = round.applications.find( + app => app.id === params.alloApplicationId ); + if (application == null) { + logger.error( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + throw new NotFoundError( + `Application with ID: ${params.alloApplicationId} not found in round` + ); + } + + roundMetadata = round.roundMetadata; + applicationMetadata = application.metadata; } - const application = round.applications.find( - app => app.id === params.alloApplicationId + const evaluation = await requestEvaluation( + roundMetadata, + applicationMetadata, + evaluationQuestions ); - if (application == null) { - logger.error( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - throw new NotFoundError( - `Application with ID: ${params.alloApplicationId} not found in round` - ); - } - roundMetadata = round.roundMetadata; - applicationMetadata = application.metadata; + await createEvaluation({ + chainId: params.chainId, + alloPoolId: params.alloPoolId, + alloApplicationId: params.alloApplicationId, + cid: params.cid, + evaluator: params.evaluator, + summaryInput: evaluation, + evaluatorType: EVALUATOR_TYPE.LLM_GPT3, + }); + } catch (error) { + // If an error occurs, add the project ID to the failedProjects array + failedProjects.push(params.alloApplicationId); + throw error; } - - const evaluation = await requestEvaluation( - roundMetadata, - applicationMetadata, - evaluationQuestions - ); - - await createEvaluation({ - chainId: params.chainId, - alloPoolId: params.alloPoolId, - alloApplicationId: params.alloApplicationId, - cid: params.cid, - evaluator: params.evaluator, - summaryInput: evaluation, - evaluatorType: EVALUATOR_TYPE.LLM_GPT3, - }); }); await Promise.all(evaluationPromises); @@ -405,4 +415,6 @@ export const createLLMEvaluations = async ( continue; } } -}; + + return failedProjects; +}; \ No newline at end of file diff --git a/src/controllers/poolController.ts b/src/controllers/poolController.ts index 66faf38..b740631 100644 --- a/src/controllers/poolController.ts +++ b/src/controllers/poolController.ts @@ -101,16 +101,34 @@ export const syncPool = async (req: Request, res: Response): Promise => { // ---- LLM evaluation ---- // Trigger LLM evaluation for the pool if there are applications without evaluations - if (skipEvaluation !== true) - await triggerLLMEvaluationByPool( + let failedProjects: string[] = []; + if (skipEvaluation !== true) { + failedProjects = await triggerLLMEvaluationByPool( alloPoolId, chainId, indexerPoolData, evaluationQuestions ); - // Log success and respond to the request - logger.info('successfully synced pool', pool); - res.status(200).json({ message: 'pool synced successfully' }); + } + + // Check if there are any failed projects and respond accordingly + if (failedProjects.length > 0) { + logger.info('Pool synced successfully with some projects failing', { + pool, + failedProjects, + }); + res.status(207).json({ + success: true, + message: 'Pool synced successfully, with some projects failing.', + failedProjects, + }); + } else { + logger.info('Successfully synced pool', pool); + res.status(200).json({ + success: true, + message: 'Pool synced successfully.', + }); + } }; const handlePoolEvaluationQuestions = async ( @@ -171,7 +189,7 @@ const triggerLLMEvaluationByPool = async ( chainId: number, indexerPoolData: IndexerRoundWithApplications, evaluationQuestions: PromptEvaluationQuestions -): Promise => { +): Promise => { const applicationsWithoutLLM = await applicationService.getApplicationsWithoutLLMEvalutionsByAlloPoolId( alloPoolId, @@ -212,5 +230,5 @@ const triggerLLMEvaluationByPool = async ( logger.warn('Some applications were not found in indexerPoolData'); } - await createLLMEvaluations(evaluationParamsArray); + return await createLLMEvaluations(evaluationParamsArray); }; diff --git a/src/ext/openai/prompt.ts b/src/ext/openai/prompt.ts index f6ad5fc..8b5ecfb 100644 --- a/src/ext/openai/prompt.ts +++ b/src/ext/openai/prompt.ts @@ -39,11 +39,11 @@ const sanitizeAndReduceMetadata = ( }; const sanitizeRoundMetadata = (metadata: RoundMetadata): string => { - return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 1000); + return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 50000); }; const sanitizeApplicationMetadata = (metadata: object): string => { - return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 3000); + return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 50000); }; export const createAiEvaluationPrompt = ( roundMetadata: RoundMetadata, @@ -52,9 +52,11 @@ export const createAiEvaluationPrompt = ( ): string => { const sanitizedRoundMetadata = sanitizeRoundMetadata(roundMetadata); const sanitizedApplicationMetadata = sanitizeApplicationMetadata( - applicationMetadata.application.project + applicationMetadata.application ); + console.log('sanitizedApplicationMetadata', sanitizedApplicationMetadata); + const questionsString = applicationQuestions .map((q, index) => `${index + 1}. ${q}`) .join('\n'); diff --git a/src/routes/poolRoutes.ts b/src/routes/poolRoutes.ts index cd1e51c..0090488 100644 --- a/src/routes/poolRoutes.ts +++ b/src/routes/poolRoutes.ts @@ -18,31 +18,43 @@ const router = Router(); * alloPoolId: * type: string * description: The ID of the pool to create - * example: "609" # Example of poolId + * example: "609" * chainId: * type: number * description: The chain ID associated with the pool - * example: 42161 # Example of chainId (Arbitrum) + * example: 42161 * skipEvaluation: - * type: boolean - * description: Skip evaluation of the pool - * example: true + * type: boolean + * description: Skip evaluation of the pool + * example: true * required: * - alloPoolId * - chainId * responses: * 201: - * description: Pool created successfully + * description: Pool synced successfully + * 207: + * description: Pool synced successfully, with some projects failing + * content: + * application/json: + * schema: + * type: object + * properties: + * success: + * type: boolean + * example: true + * message: + * type: string + * example: Pool synced successfully, with some projects failing. + * failedProjects: + * type: array + * items: + * type: string + * example: "0xprojectId1" * 400: * description: Invalid poolId or chainId format * 500: * description: Internal server error - * examples: - * application/json: - * - value: - * alloPoolId: "609" - * chainId: "42161" - * skipEvaluation: true */ router.post('/', syncPool); From 774c4e95382b861b2bd5058a3f1e7392a5a8b67b Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Thu, 5 Dec 2024 13:48:43 +0100 Subject: [PATCH 2/4] pretty --- src/controllers/evaluationController.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/evaluationController.ts b/src/controllers/evaluationController.ts index d2f00c2..80451d7 100644 --- a/src/controllers/evaluationController.ts +++ b/src/controllers/evaluationController.ts @@ -417,4 +417,4 @@ export const createLLMEvaluations = async ( } return failedProjects; -}; \ No newline at end of file +}; From e6da83ba93c6a898b22cf822d36e33c9e032ac86 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Thu, 5 Dec 2024 13:50:53 +0100 Subject: [PATCH 3/4] rm log --- evalChecker.js | 169 ++++++++++++++++++++++++++++++++++ evalChecker.ts | 194 +++++++++++++++++++++++++++++++++++++++ src/ext/openai/prompt.ts | 2 - 3 files changed, 363 insertions(+), 2 deletions(-) create mode 100755 evalChecker.js create mode 100755 evalChecker.ts diff --git a/evalChecker.js b/evalChecker.js new file mode 100755 index 0000000..be8dadc --- /dev/null +++ b/evalChecker.js @@ -0,0 +1,169 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +var axios_1 = require("axios"); +// Function to make a GraphQL request and return the response data +function fetchGraphQL(endpoint, query) { + return __awaiter(this, void 0, void 0, function () { + var response; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, axios_1.default.post(endpoint, { query: query })]; + case 1: + response = _a.sent(); + return [2 /*return*/, response.data.data]; + } + }); + }); +} +// Function to compare human and AI evaluations for a specific pool +function compareEvaluations(chainId, poolId) { + return __awaiter(this, void 0, void 0, function () { + var humanEvalServer, aiEvalServer, humanEvaluationQuery, aiEvaluationQuery, humanData, aiData, humanApplications, aiApplications_1, uncertainApplications_1, matchingApplications_1, nonMatchingApplications_1, error_1; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + _a.trys.push([0, 3, , 4]); + humanEvalServer = 'https://grants-stack-indexer-v2.gitcoin.co/graphql'; + aiEvalServer = 'https://api.checker.gitcoin.co/graphql'; + humanEvaluationQuery = "\n query MyQuery {\n round(chainId: ".concat(chainId, ", id: \"").concat(poolId, "\") {\n applications {\n id\n status\n }\n }\n }\n "); + aiEvaluationQuery = "\n query MyQuery {\n applications(filter: {pool: {alloPoolId: {equalTo: \"".concat(poolId, "\"}}}) {\n alloApplicationId\n evaluations {\n evaluationStatus\n }\n }\n }\n "); + return [4 /*yield*/, fetchGraphQL(humanEvalServer, humanEvaluationQuery)]; + case 1: + humanData = _a.sent(); + return [4 /*yield*/, fetchGraphQL(aiEvalServer, aiEvaluationQuery)]; + case 2: + aiData = _a.sent(); + humanApplications = humanData.round.applications; + aiApplications_1 = aiData.applications; + uncertainApplications_1 = []; + matchingApplications_1 = []; + nonMatchingApplications_1 = []; + // Compare human applications with AI applications + humanApplications.forEach(function (humanApp) { + var aiApp = aiApplications_1.find(function (aiApp) { return aiApp.alloApplicationId === humanApp.id; }); + if (aiApp !== undefined && aiApp.evaluations.length > 0) { + var aiStatus = aiApp.evaluations[0].evaluationStatus; + if (aiStatus === humanApp.status) { + // If AI evaluation exists and status matches humanApp.status, push to matching list + matchingApplications_1.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus: aiStatus, + }); + } + else if (aiStatus === 'UNCERTAIN') { + // If AI evaluation status is uncertain, push to uncertain list + uncertainApplications_1.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus: aiStatus, + }); + } + else { + // If evaluation exists but status does not match, push to non-matching list + nonMatchingApplications_1.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus: aiStatus, + }); + } + } + else { + // If no evaluation exists, push to non-matching list + nonMatchingApplications_1.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus: 'No Evaluation', + }); + } + }); + // Print the results for the current pool + console.log("Results for Pool ".concat(poolId, " (Chain ID: ").concat(chainId, "):")); + console.log("Matching Applications: ".concat(matchingApplications_1.length, "/").concat(humanApplications.length)); + matchingApplications_1.forEach(function (app) { + console.log("ID: ".concat(app.id, ", Human Status: ").concat(app.humanStatus, ", AI Status: ").concat(app.aiStatus)); + }); + console.log("Uncertain Applications: ".concat(uncertainApplications_1.length, "/").concat(humanApplications.length)); + uncertainApplications_1.forEach(function (app) { + console.log("ID: ".concat(app.id, ", Human Status: ").concat(app.humanStatus, ", AI Status: ").concat(app.aiStatus)); + }); + console.log("Non-Matching Applications: ".concat(nonMatchingApplications_1.length, "/").concat(humanApplications.length)); + nonMatchingApplications_1.forEach(function (app) { + console.log("ID: ".concat(app.id, ", Human Status: ").concat(app.humanStatus, ", AI Status: ").concat(app.aiStatus)); + }); + console.log('\n\n\n---\n\n\n'); + return [3 /*break*/, 4]; + case 3: + error_1 = _a.sent(); + console.error('Error fetching data from servers:', error_1); + throw new Error('Failed to compare evaluations'); + case 4: return [2 /*return*/]; + } + }); + }); +} +// Iterate through an array of pool and chainId pairs +var poolsToCheck = [ + { chainId: 42161, poolId: '608' }, + { chainId: 42161, poolId: '609' }, + { chainId: 42161, poolId: '610' }, + { chainId: 42161, poolId: '611' }, + // Add more pools as needed +]; +void (function () { return __awaiter(void 0, void 0, void 0, function () { + var _i, poolsToCheck_1, _a, chainId, poolId; + return __generator(this, function (_b) { + switch (_b.label) { + case 0: + _i = 0, poolsToCheck_1 = poolsToCheck; + _b.label = 1; + case 1: + if (!(_i < poolsToCheck_1.length)) return [3 /*break*/, 4]; + _a = poolsToCheck_1[_i], chainId = _a.chainId, poolId = _a.poolId; + return [4 /*yield*/, compareEvaluations(chainId, poolId)]; + case 2: + _b.sent(); + _b.label = 3; + case 3: + _i++; + return [3 /*break*/, 1]; + case 4: return [2 /*return*/]; + } + }); +}); })(); diff --git a/evalChecker.ts b/evalChecker.ts new file mode 100755 index 0000000..475fc68 --- /dev/null +++ b/evalChecker.ts @@ -0,0 +1,194 @@ +import axios from 'axios'; + +// Define types for the human and AI evaluation queries' responses +interface HumanEvaluationApplication { + id: string; + status: string; +} + +interface HumanEvaluationResponse { + round: { + applications: HumanEvaluationApplication[]; + }; +} + +interface AIEvaluation { + evaluationStatus: string; +} + +interface AIEvaluationApplication { + alloApplicationId: string; + evaluations: AIEvaluation[]; +} + +interface AIEvaluationResponse { + applications: AIEvaluationApplication[]; +} + +// Define types for the results we are comparing +interface MatchingApplication { + id: string; + humanStatus: string; + aiStatus: string; +} + +interface NonMatchingApplication { + id: string; + humanStatus: string; + aiStatus: string; +} + +interface UncertainApplication { + id: string; + humanStatus: string; + aiStatus: string; +} + +// Function to make a GraphQL request and return the response data +async function fetchGraphQL(endpoint: string, query: string): Promise { + const response = await axios.post(endpoint, { query }); + return response.data.data; +} + +// Function to compare human and AI evaluations for a specific pool +async function compareEvaluations( + chainId: number, + poolId: string +): Promise { + try { + // URLs of the two GraphQL servers + const humanEvalServer = + 'https://grants-stack-indexer-v2.gitcoin.co/graphql'; + const aiEvalServer = 'https://api.checker.gitcoin.co/graphql'; + + // Define GraphQL queries dynamically based on chainId and poolId + const humanEvaluationQuery = ` + query MyQuery { + round(chainId: ${chainId}, id: "${poolId}") { + applications { + id + status + } + } + } + `; + + const aiEvaluationQuery = ` + query MyQuery { + applications(filter: {pool: {alloPoolId: {equalTo: "${poolId}"}}}) { + alloApplicationId + evaluations { + evaluationStatus + } + } + } + `; + + // Fetch data from both servers + const humanData = await fetchGraphQL( + humanEvalServer, + humanEvaluationQuery + ); + const aiData = await fetchGraphQL( + aiEvalServer, + aiEvaluationQuery + ); + + // Extract the relevant application data + const humanApplications = humanData.round.applications; + const aiApplications = aiData.applications; + + // Initialize lists for matching, non-matching, and uncertain results + const uncertainApplications: UncertainApplication[] = []; + const matchingApplications: MatchingApplication[] = []; + const nonMatchingApplications: NonMatchingApplication[] = []; + + // Compare human applications with AI applications + humanApplications.forEach(humanApp => { + const aiApp = aiApplications.find( + aiApp => aiApp.alloApplicationId === humanApp.id + ); + + if (aiApp !== undefined && aiApp.evaluations.length > 0) { + const aiStatus = aiApp.evaluations[0].evaluationStatus; + + if (aiStatus === humanApp.status) { + // If AI evaluation exists and status matches humanApp.status, push to matching list + matchingApplications.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus, + }); + } else if (aiStatus === 'UNCERTAIN') { + // If AI evaluation status is uncertain, push to uncertain list + uncertainApplications.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus, + }); + } else { + // If evaluation exists but status does not match, push to non-matching list + nonMatchingApplications.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus, + }); + } + } else { + // If no evaluation exists, push to non-matching list + nonMatchingApplications.push({ + id: humanApp.id, + humanStatus: humanApp.status, + aiStatus: 'No Evaluation', + }); + } + }); + + // Print the results for the current pool + console.log(`Results for Pool ${poolId} (Chain ID: ${chainId}):`); + console.log( + `Matching Applications: ${matchingApplications.length}/${humanApplications.length}` + ); + matchingApplications.forEach(app => { + console.log( + `ID: ${app.id}, Human Status: ${app.humanStatus}, AI Status: ${app.aiStatus}` + ); + }); + console.log( + `Uncertain Applications: ${uncertainApplications.length}/${humanApplications.length}` + ); + uncertainApplications.forEach(app => { + console.log( + `ID: ${app.id}, Human Status: ${app.humanStatus}, AI Status: ${app.aiStatus}` + ); + }); + console.log( + `Non-Matching Applications: ${nonMatchingApplications.length}/${humanApplications.length}` + ); + nonMatchingApplications.forEach(app => { + console.log( + `ID: ${app.id}, Human Status: ${app.humanStatus}, AI Status: ${app.aiStatus}` + ); + }); + console.log('\n\n\n---\n\n\n'); + } catch (error) { + console.error('Error fetching data from servers:', error); + throw new Error('Failed to compare evaluations'); + } +} + +// Iterate through an array of pool and chainId pairs +const poolsToCheck = [ + { chainId: 42161, poolId: '608' }, + { chainId: 42161, poolId: '609' }, + { chainId: 42161, poolId: '610' }, + { chainId: 42161, poolId: '611' }, + + // Add more pools as needed +]; + +void (async () => { + for (const { chainId, poolId } of poolsToCheck) { + await compareEvaluations(chainId, poolId); + } +})(); diff --git a/src/ext/openai/prompt.ts b/src/ext/openai/prompt.ts index 8b5ecfb..a335f13 100644 --- a/src/ext/openai/prompt.ts +++ b/src/ext/openai/prompt.ts @@ -55,8 +55,6 @@ export const createAiEvaluationPrompt = ( applicationMetadata.application ); - console.log('sanitizedApplicationMetadata', sanitizedApplicationMetadata); - const questionsString = applicationQuestions .map((q, index) => `${index + 1}. ${q}`) .join('\n'); From f561434af276f40cc4ad11b96af3ecc9087431f9 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Thu, 5 Dec 2024 14:03:16 +0100 Subject: [PATCH 4/4] rm files --- evalChecker.js | 169 ------------------------------------------ evalChecker.ts | 194 ------------------------------------------------- 2 files changed, 363 deletions(-) delete mode 100755 evalChecker.js delete mode 100755 evalChecker.ts diff --git a/evalChecker.js b/evalChecker.js deleted file mode 100755 index be8dadc..0000000 --- a/evalChecker.js +++ /dev/null @@ -1,169 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __generator = (this && this.__generator) || function (thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; - return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (g && (g = 0, op[0] && (_ = 0)), _) try { - if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -var axios_1 = require("axios"); -// Function to make a GraphQL request and return the response data -function fetchGraphQL(endpoint, query) { - return __awaiter(this, void 0, void 0, function () { - var response; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, axios_1.default.post(endpoint, { query: query })]; - case 1: - response = _a.sent(); - return [2 /*return*/, response.data.data]; - } - }); - }); -} -// Function to compare human and AI evaluations for a specific pool -function compareEvaluations(chainId, poolId) { - return __awaiter(this, void 0, void 0, function () { - var humanEvalServer, aiEvalServer, humanEvaluationQuery, aiEvaluationQuery, humanData, aiData, humanApplications, aiApplications_1, uncertainApplications_1, matchingApplications_1, nonMatchingApplications_1, error_1; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - _a.trys.push([0, 3, , 4]); - humanEvalServer = 'https://grants-stack-indexer-v2.gitcoin.co/graphql'; - aiEvalServer = 'https://api.checker.gitcoin.co/graphql'; - humanEvaluationQuery = "\n query MyQuery {\n round(chainId: ".concat(chainId, ", id: \"").concat(poolId, "\") {\n applications {\n id\n status\n }\n }\n }\n "); - aiEvaluationQuery = "\n query MyQuery {\n applications(filter: {pool: {alloPoolId: {equalTo: \"".concat(poolId, "\"}}}) {\n alloApplicationId\n evaluations {\n evaluationStatus\n }\n }\n }\n "); - return [4 /*yield*/, fetchGraphQL(humanEvalServer, humanEvaluationQuery)]; - case 1: - humanData = _a.sent(); - return [4 /*yield*/, fetchGraphQL(aiEvalServer, aiEvaluationQuery)]; - case 2: - aiData = _a.sent(); - humanApplications = humanData.round.applications; - aiApplications_1 = aiData.applications; - uncertainApplications_1 = []; - matchingApplications_1 = []; - nonMatchingApplications_1 = []; - // Compare human applications with AI applications - humanApplications.forEach(function (humanApp) { - var aiApp = aiApplications_1.find(function (aiApp) { return aiApp.alloApplicationId === humanApp.id; }); - if (aiApp !== undefined && aiApp.evaluations.length > 0) { - var aiStatus = aiApp.evaluations[0].evaluationStatus; - if (aiStatus === humanApp.status) { - // If AI evaluation exists and status matches humanApp.status, push to matching list - matchingApplications_1.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus: aiStatus, - }); - } - else if (aiStatus === 'UNCERTAIN') { - // If AI evaluation status is uncertain, push to uncertain list - uncertainApplications_1.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus: aiStatus, - }); - } - else { - // If evaluation exists but status does not match, push to non-matching list - nonMatchingApplications_1.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus: aiStatus, - }); - } - } - else { - // If no evaluation exists, push to non-matching list - nonMatchingApplications_1.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus: 'No Evaluation', - }); - } - }); - // Print the results for the current pool - console.log("Results for Pool ".concat(poolId, " (Chain ID: ").concat(chainId, "):")); - console.log("Matching Applications: ".concat(matchingApplications_1.length, "/").concat(humanApplications.length)); - matchingApplications_1.forEach(function (app) { - console.log("ID: ".concat(app.id, ", Human Status: ").concat(app.humanStatus, ", AI Status: ").concat(app.aiStatus)); - }); - console.log("Uncertain Applications: ".concat(uncertainApplications_1.length, "/").concat(humanApplications.length)); - uncertainApplications_1.forEach(function (app) { - console.log("ID: ".concat(app.id, ", Human Status: ").concat(app.humanStatus, ", AI Status: ").concat(app.aiStatus)); - }); - console.log("Non-Matching Applications: ".concat(nonMatchingApplications_1.length, "/").concat(humanApplications.length)); - nonMatchingApplications_1.forEach(function (app) { - console.log("ID: ".concat(app.id, ", Human Status: ").concat(app.humanStatus, ", AI Status: ").concat(app.aiStatus)); - }); - console.log('\n\n\n---\n\n\n'); - return [3 /*break*/, 4]; - case 3: - error_1 = _a.sent(); - console.error('Error fetching data from servers:', error_1); - throw new Error('Failed to compare evaluations'); - case 4: return [2 /*return*/]; - } - }); - }); -} -// Iterate through an array of pool and chainId pairs -var poolsToCheck = [ - { chainId: 42161, poolId: '608' }, - { chainId: 42161, poolId: '609' }, - { chainId: 42161, poolId: '610' }, - { chainId: 42161, poolId: '611' }, - // Add more pools as needed -]; -void (function () { return __awaiter(void 0, void 0, void 0, function () { - var _i, poolsToCheck_1, _a, chainId, poolId; - return __generator(this, function (_b) { - switch (_b.label) { - case 0: - _i = 0, poolsToCheck_1 = poolsToCheck; - _b.label = 1; - case 1: - if (!(_i < poolsToCheck_1.length)) return [3 /*break*/, 4]; - _a = poolsToCheck_1[_i], chainId = _a.chainId, poolId = _a.poolId; - return [4 /*yield*/, compareEvaluations(chainId, poolId)]; - case 2: - _b.sent(); - _b.label = 3; - case 3: - _i++; - return [3 /*break*/, 1]; - case 4: return [2 /*return*/]; - } - }); -}); })(); diff --git a/evalChecker.ts b/evalChecker.ts deleted file mode 100755 index 475fc68..0000000 --- a/evalChecker.ts +++ /dev/null @@ -1,194 +0,0 @@ -import axios from 'axios'; - -// Define types for the human and AI evaluation queries' responses -interface HumanEvaluationApplication { - id: string; - status: string; -} - -interface HumanEvaluationResponse { - round: { - applications: HumanEvaluationApplication[]; - }; -} - -interface AIEvaluation { - evaluationStatus: string; -} - -interface AIEvaluationApplication { - alloApplicationId: string; - evaluations: AIEvaluation[]; -} - -interface AIEvaluationResponse { - applications: AIEvaluationApplication[]; -} - -// Define types for the results we are comparing -interface MatchingApplication { - id: string; - humanStatus: string; - aiStatus: string; -} - -interface NonMatchingApplication { - id: string; - humanStatus: string; - aiStatus: string; -} - -interface UncertainApplication { - id: string; - humanStatus: string; - aiStatus: string; -} - -// Function to make a GraphQL request and return the response data -async function fetchGraphQL(endpoint: string, query: string): Promise { - const response = await axios.post(endpoint, { query }); - return response.data.data; -} - -// Function to compare human and AI evaluations for a specific pool -async function compareEvaluations( - chainId: number, - poolId: string -): Promise { - try { - // URLs of the two GraphQL servers - const humanEvalServer = - 'https://grants-stack-indexer-v2.gitcoin.co/graphql'; - const aiEvalServer = 'https://api.checker.gitcoin.co/graphql'; - - // Define GraphQL queries dynamically based on chainId and poolId - const humanEvaluationQuery = ` - query MyQuery { - round(chainId: ${chainId}, id: "${poolId}") { - applications { - id - status - } - } - } - `; - - const aiEvaluationQuery = ` - query MyQuery { - applications(filter: {pool: {alloPoolId: {equalTo: "${poolId}"}}}) { - alloApplicationId - evaluations { - evaluationStatus - } - } - } - `; - - // Fetch data from both servers - const humanData = await fetchGraphQL( - humanEvalServer, - humanEvaluationQuery - ); - const aiData = await fetchGraphQL( - aiEvalServer, - aiEvaluationQuery - ); - - // Extract the relevant application data - const humanApplications = humanData.round.applications; - const aiApplications = aiData.applications; - - // Initialize lists for matching, non-matching, and uncertain results - const uncertainApplications: UncertainApplication[] = []; - const matchingApplications: MatchingApplication[] = []; - const nonMatchingApplications: NonMatchingApplication[] = []; - - // Compare human applications with AI applications - humanApplications.forEach(humanApp => { - const aiApp = aiApplications.find( - aiApp => aiApp.alloApplicationId === humanApp.id - ); - - if (aiApp !== undefined && aiApp.evaluations.length > 0) { - const aiStatus = aiApp.evaluations[0].evaluationStatus; - - if (aiStatus === humanApp.status) { - // If AI evaluation exists and status matches humanApp.status, push to matching list - matchingApplications.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus, - }); - } else if (aiStatus === 'UNCERTAIN') { - // If AI evaluation status is uncertain, push to uncertain list - uncertainApplications.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus, - }); - } else { - // If evaluation exists but status does not match, push to non-matching list - nonMatchingApplications.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus, - }); - } - } else { - // If no evaluation exists, push to non-matching list - nonMatchingApplications.push({ - id: humanApp.id, - humanStatus: humanApp.status, - aiStatus: 'No Evaluation', - }); - } - }); - - // Print the results for the current pool - console.log(`Results for Pool ${poolId} (Chain ID: ${chainId}):`); - console.log( - `Matching Applications: ${matchingApplications.length}/${humanApplications.length}` - ); - matchingApplications.forEach(app => { - console.log( - `ID: ${app.id}, Human Status: ${app.humanStatus}, AI Status: ${app.aiStatus}` - ); - }); - console.log( - `Uncertain Applications: ${uncertainApplications.length}/${humanApplications.length}` - ); - uncertainApplications.forEach(app => { - console.log( - `ID: ${app.id}, Human Status: ${app.humanStatus}, AI Status: ${app.aiStatus}` - ); - }); - console.log( - `Non-Matching Applications: ${nonMatchingApplications.length}/${humanApplications.length}` - ); - nonMatchingApplications.forEach(app => { - console.log( - `ID: ${app.id}, Human Status: ${app.humanStatus}, AI Status: ${app.aiStatus}` - ); - }); - console.log('\n\n\n---\n\n\n'); - } catch (error) { - console.error('Error fetching data from servers:', error); - throw new Error('Failed to compare evaluations'); - } -} - -// Iterate through an array of pool and chainId pairs -const poolsToCheck = [ - { chainId: 42161, poolId: '608' }, - { chainId: 42161, poolId: '609' }, - { chainId: 42161, poolId: '610' }, - { chainId: 42161, poolId: '611' }, - - // Add more pools as needed -]; - -void (async () => { - for (const { chainId, poolId } of poolsToCheck) { - await compareEvaluations(chainId, poolId); - } -})();