From b8664969e5151e979a9334bbf7d5076dedb89c1c Mon Sep 17 00:00:00 2001 From: noaha Date: Wed, 8 Jan 2025 15:38:00 -0500 Subject: [PATCH 1/5] fix: getField * Fix bug which skips the first column regardless of if it is a timestamp * date(timestamp + "Z") was breaking timestamps. Use whatever O2 gives us, unless it's microseconds which needs converting to MS. * Only add Time column if it's part of the request [breaking for queries relying on this implicit field, but deduplicating for the queries that manually request it including example requests] --- src/features/log/queryResponseBuilder.ts | 28 +++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/features/log/queryResponseBuilder.ts b/src/features/log/queryResponseBuilder.ts index b1eca07..b7a5c2a 100644 --- a/src/features/log/queryResponseBuilder.ts +++ b/src/features/log/queryResponseBuilder.ts @@ -69,19 +69,31 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { } data.forEach((log: any) => { - graphData.add(getField(log, fields)); + graphData.add(getField(log, fields, '_timestamp')); }); return graphData; }; -const getField = (log: any, columns: any) => { - let field: any = { - Time: new Date(log[columns[0]] + 'Z').getTime(), - }; - - for (let i = 1; i < columns.length; i++) { - field[columns[i]] = log[columns[i]]; +const getField = (log: any, columns: any, timestampColumn: string) => { + let field: any = {}; + + for (let i = 0; i < columns.length; i++) { + let col_name = columns[i]; + let col_value = log[col_name] + if (col_name === timestampColumn) { + // We have to convert microseconds if we receive them + // 500 billion / year 17814 is probably a good threshold for milliseconds + if (col_value > 500_000_000_000) { + col_value = convertTimeToMs(col_value); + field["Time"] = col_value; + } else { + // Convert any other date fmt + field["Time"] = new Date(col_value).getTime(); + } + } else { + field[col_name] = log[col_name]; + } } return field; From 31a8d3395823d46ad9f99d3720282a435254793b Mon Sep 17 00:00:00 2001 From: noaha Date: Wed, 8 Jan 2025 15:41:01 -0500 Subject: [PATCH 2/5] fix: getGraphDataFrame * [bugfix] No longer skip first requested column * [bugfix] Remove type: number for non-time fields which breaks string responses * Detect timestamp column in request --- src/features/log/queryResponseBuilder.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/features/log/queryResponseBuilder.ts b/src/features/log/queryResponseBuilder.ts index b7a5c2a..48ec4db 100644 --- a/src/features/log/queryResponseBuilder.ts +++ b/src/features/log/queryResponseBuilder.ts @@ -49,6 +49,8 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { } } + for (let i = 0; i < fields.length; i++) { + if (fields[i] === '_timestamp') { graphData.addField({ config: { filterable: true, @@ -56,12 +58,11 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { name: 'Time', type: FieldType.time, }); - - for (let i = 1; i < fields.length; i++) { + } else { graphData.addField({ name: fields[i], - type: FieldType.number, }); + } } if (!data.length) { From 0df4c9f6a8b2fdd2a993181c7a824e6fb788814e Mon Sep 17 00:00:00 2001 From: noaha Date: Wed, 8 Jan 2025 15:43:56 -0500 Subject: [PATCH 3/5] fix: Add getColumnsFromQuery quote stripping [big] The dataframe parser would fail to assign fields wrapped in quotes i.e. select 1 as "server" --- src/features/log/queryResponseBuilder.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/features/log/queryResponseBuilder.ts b/src/features/log/queryResponseBuilder.ts index 48ec4db..498e879 100644 --- a/src/features/log/queryResponseBuilder.ts +++ b/src/features/log/queryResponseBuilder.ts @@ -51,17 +51,17 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { for (let i = 0; i < fields.length; i++) { if (fields[i] === '_timestamp') { - graphData.addField({ - config: { - filterable: true, - }, - name: 'Time', - type: FieldType.time, - }); + graphData.addField({ + config: { + filterable: true, + }, + name: 'Time', + type: FieldType.time, + }); } else { - graphData.addField({ - name: fields[i], - }); + graphData.addField({ + name: fields[i], + }); } } @@ -140,7 +140,9 @@ const getColumnsFromQuery = (query: string) => { // If alias exists, use that, otherwise use column name if (aliasMatch) { - columnNames.push(aliasMatch[1]); + // SQL alias may have quotes, strip those. + let stripped = aliasMatch[1].replace(/^['"]|['"]$/g, ''); + columnNames.push(stripped); } else { columnNames.push(column); } From b3b10557c2a3000dc7ee8f28334298150a200d2e Mon Sep 17 00:00:00 2001 From: noaha Date: Wed, 8 Jan 2025 15:46:38 -0500 Subject: [PATCH 4/5] change: Utilize datasource defined timestamp column automatically in data frame parser --- src/datasource.ts | 6 +++--- src/features/log/queryResponseBuilder.ts | 11 ++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/datasource.ts b/src/datasource.ts index 47b35e9..6b0fbad 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -72,7 +72,7 @@ export class DataSource // As we don't show histogram for sql mode in explore if (options.app === 'explore' && target?.refId?.includes(REF_ID_STARTER_LOG_VOLUME) && target.sqlMode) { - return getGraphDataFrame([], target, options.app); + return getGraphDataFrame([], target, options.app, this.timestampColumn); } this.cachedQuery.requestQuery = JSON.stringify(reqData); @@ -80,12 +80,12 @@ export class DataSource return this.doRequest(target, reqData) .then((response) => { if (options.app === 'panel-editor' || options.app === 'dashboard') { - return getGraphDataFrame(response.hits, target, options.app); + return getGraphDataFrame(response.hits, target, options.app, this.timestampColumn); } const logsDataFrame = getLogsDataFrame(response.hits, target, this.streamFields, this.timestampColumn); - const graphDataFrame = getGraphDataFrame(response?.aggs?.histogram || [], target, options.app); + const graphDataFrame = getGraphDataFrame(response?.aggs?.histogram || [], target, options.app, this.timestampColumn); this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: logsDataFrame }); diff --git a/src/features/log/queryResponseBuilder.ts b/src/features/log/queryResponseBuilder.ts index 498e879..8c4600b 100644 --- a/src/features/log/queryResponseBuilder.ts +++ b/src/features/log/queryResponseBuilder.ts @@ -36,7 +36,12 @@ export const getLogsDataFrame = ( return logsData; }; -export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { +export const getGraphDataFrame = ( + data: any, + target: MyQuery, + app: string, + timestampColumn = '_timestamp' +) => { const graphData = getDefaultDataFrame(target.refId, 'graph'); let fields = ['zo_sql_key', 'zo_sql_num']; @@ -50,7 +55,7 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { } for (let i = 0; i < fields.length; i++) { - if (fields[i] === '_timestamp') { + if (fields[i] === timestampColumn) { graphData.addField({ config: { filterable: true, @@ -70,7 +75,7 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => { } data.forEach((log: any) => { - graphData.add(getField(log, fields, '_timestamp')); + graphData.add(getField(log, fields, timestampColumn)); }); return graphData; From 8561dab327348cb1dffc5b366a1a9d511b827069 Mon Sep 17 00:00:00 2001 From: noaha Date: Wed, 8 Jan 2025 15:48:36 -0500 Subject: [PATCH 5/5] Implement #14 fix in #15 to pass tests --- package.json | 2 +- src/components/QueryEditor.tsx | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/package.json b/package.json index 3c828ec..09fe31a 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "eslint-plugin-deprecation": "^2.0.0" }, "engines": { - "node": ">=16" + "node": ">=18" }, "dependencies": { "@emotion/css": "^11.1.3", diff --git a/src/components/QueryEditor.tsx b/src/components/QueryEditor.tsx index 87c77e2..734ecb4 100644 --- a/src/components/QueryEditor.tsx +++ b/src/components/QueryEditor.tsx @@ -32,12 +32,6 @@ export const QueryEditor = ({ query, onChange, onRunQuery, datasource, app, data setIsLoading(isLoading.slice(1)); }; - const isInDashboard = useMemo(() => app === 'panel-editor', [app]); - - const getTimeStampColumnName = () => { - return datasource.instanceSettings?.jsonData?.timestamp_column || '_timestamp'; - }; - useEffect(() => { startLoading(); getOrganizations({ url: datasource.url, page_num: 0, page_size: 1000, sort_by: 'id' })