From 4978f5a5a492dfa8425da618ea212e463a28c0d8 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 15 Aug 2025 14:05:08 +0000 Subject: [PATCH 1/2] latest --- .../src/routes/api/presto-search/index.ts | 43 +++++++++++++------ .../src/routes/api/presto-search/typings.ts | 4 ++ 2 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 components/webui/server/src/routes/api/presto-search/typings.ts diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 1f714835fb..44f2640465 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -16,7 +16,7 @@ import { PrestoQueryJobSchema, } from "../../../schemas/presto-search.js"; import {insertPrestoRowsToMongo} from "./utils.js"; - +import {MAX_PRESTO_SEARCH_RESULTS} from "./typings.js"; /** * Presto search API routes. @@ -61,18 +61,19 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const {queryString} = request.body; let searchJobId: string; + let totalResultsCount = 0; + let storedResultsCount = 0; try { // eslint-disable-next-line max-lines-per-function searchJobId = await new Promise((resolve, reject) => { let isResolved = false; Presto.client.execute({ - // eslint-disable-next-line no-warning-comments - // TODO: Error, and success handlers are dummy implementations - // and will be replaced with proper implementations. data: (_, data, columns) => { + totalResultsCount += data.length; + request.log.info( - `Received ${data.length} rows from Presto query` + `Received ${data.length} rows from Presto query (total: ${totalResultsCount})` ); if (false === isResolved) { @@ -80,7 +81,6 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { "Presto data received before searchJobId was resolved; " + "skipping insert." ); - return; } @@ -88,15 +88,34 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { return; } - insertPrestoRowsToMongo( - data, - columns, - searchJobId, - mongoDb + if (storedResultsCount < MAX_PRESTO_SEARCH_RESULTS) { + const remainingSlots = MAX_PRESTO_SEARCH_RESULTS - storedResultsCount; + const dataToInsert = data.slice(0, remainingSlots); + + if (dataToInsert.length > 0) { + storedResultsCount += dataToInsert.length; + insertPrestoRowsToMongo( + dataToInsert, + columns, + searchJobId, + mongoDb + ).catch((err: unknown) => { + request.log.error( + err, + "Failed to insert Presto results into MongoDB" + ); + }); + } + } + + // Always update metadata with total count + searchResultsMetadataCollection.updateOne( + {_id: searchJobId}, + {$set: {numTotalResults: totalResultsCount}} ).catch((err: unknown) => { request.log.error( err, - "Failed to insert Presto results into MongoDB" + "Failed to update total results count in metadata" ); }); }, diff --git a/components/webui/server/src/routes/api/presto-search/typings.ts b/components/webui/server/src/routes/api/presto-search/typings.ts new file mode 100644 index 0000000000..bdf72a8b87 --- /dev/null +++ b/components/webui/server/src/routes/api/presto-search/typings.ts @@ -0,0 +1,4 @@ +/** + * Maximum number of Presto search results to store in MongoDB. + */ +export const MAX_PRESTO_SEARCH_RESULTS = 1000; From 96be3261280a8051ab2f80d058002046f827d95a Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 15 Aug 2025 14:21:56 +0000 Subject: [PATCH 2/2] latest --- .../server/src/routes/api/presto-search/index.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 44f2640465..ad0dd3f361 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -15,8 +15,9 @@ import { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, } from "../../../schemas/presto-search.js"; -import {insertPrestoRowsToMongo} from "./utils.js"; import {MAX_PRESTO_SEARCH_RESULTS} from "./typings.js"; +import {insertPrestoRowsToMongo} from "./utils.js"; + /** * Presto search API routes. @@ -73,7 +74,8 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { totalResultsCount += data.length; request.log.info( - `Received ${data.length} rows from Presto query (total: ${totalResultsCount})` + `Received ${data.length} rows from Presto query ` + + `(total: ${totalResultsCount})` ); if (false === isResolved) { @@ -81,6 +83,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { "Presto data received before searchJobId was resolved; " + "skipping insert." ); + return; } @@ -89,10 +92,11 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { } if (storedResultsCount < MAX_PRESTO_SEARCH_RESULTS) { - const remainingSlots = MAX_PRESTO_SEARCH_RESULTS - storedResultsCount; + const remainingSlots = + MAX_PRESTO_SEARCH_RESULTS - storedResultsCount; const dataToInsert = data.slice(0, remainingSlots); - if (dataToInsert.length > 0) { + if (0 < dataToInsert.length) { storedResultsCount += dataToInsert.length; insertPrestoRowsToMongo( dataToInsert,