From 247813f8e79be2e8d0bb20dbb7c4a86534fd74cc Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 20 May 2025 17:28:26 +0300 Subject: [PATCH 01/24] prepare test for non-match because of join tree difference --- .../postgres/pre-aggregations.test.ts | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index bf603e2995947..f862859fc62cf 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -273,7 +273,7 @@ describe('PreAggregations', () => { measures: [count], dimensions: [visitor_checkins.source], timeDimension: createdAt, - granularity: 'day', + granularity: 'day' } } }) @@ -378,6 +378,13 @@ describe('PreAggregations', () => { select * from cards \`, + joins: { + visitor_checkins: { + relationship: 'one_to_many', + sql: \`\${CUBE.visitorId} = \${visitor_checkins.visitor_id}\` + } + }, + measures: { count: { type: 'count' @@ -519,6 +526,23 @@ describe('PreAggregations', () => { includes: '*' }] }); + + view('cards_visitors_checkins_view', { + cubes: [ + { + join_path: visitors, + includes: ['count', 'createdAt'] + }, + { + join_path: visitors.cards, + includes: [{ name: 'visitorId', alias: 'visitorIdFromCards'}] + }, + { + join_path: visitors.cards.visitor_checkins, + includes: ['source'] + } + ] + }); `); it('simple pre-aggregation', async () => { @@ -1187,6 +1211,64 @@ describe('PreAggregations', () => { }); }); + it('non-match because of join tree difference', async () => { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'cards_visitors_checkins_view.count' + ], + dimensions: ['cards_visitors_checkins_view.source'], + timeDimensions: [{ + dimension: 'cards_visitors_checkins_view.createdAt', + granularity: 'day', + dateRange: ['2017-01-01', '2017-01-30'] + }], + order: [{ + id: 'cards_visitors_checkins_view.createdAt' + }, { + id: 'cards_visitors_checkins_view.source' + }], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect((query).preAggregations.preAggregationForQuery).toBeUndefined(); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [ + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-02T00:00:00.000Z', + cards_visitors_checkins_view__source: 'google', + }, + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-02T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-04T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-05T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + { + cards_visitors_checkins_view__count: '2', + cards_visitors_checkins_view__created_at_day: '2017-01-06T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + ] + ); + }); + }); + it('non-leaf additive measure', async () => { await compiler.compile(); From 34037c106fd21dbcb798455f5365044cfdae8dec Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 20 May 2025 20:39:52 +0300 Subject: [PATCH 02/24] remove unused --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index dcd5978877610..b3ad04d47fde0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -715,8 +715,6 @@ export class BaseQuery { multipliedMeasures, regularMeasures, cumulativeMeasures, - withQueries, - multiStageMembers, } = this.fullKeyQueryAggregateMeasures(); if (cumulativeMeasures.length === 0) { From 1230660eaf28fde00a8eeceb77e0a4996a96adf2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 20 May 2025 21:51:10 +0300 Subject: [PATCH 03/24] fix some types --- .../src/adapter/PreAggregations.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0dd6fad25ec4f..4effc3935227a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -674,21 +674,16 @@ export class PreAggregations { /** * Expand granularity into array of granularity hierarchy. - * @param {string} dimension Dimension in the form of `cube.timeDimension` - * @param {string} granularity Granularity - * @returns {Array} */ - const expandGranularity = (dimension, granularity) => ( + const expandGranularity = (dimension: string, granularity: string): Array => ( transformedQuery.granularityHierarchies[`${dimension}.${granularity}`] || [granularity] ); /** * Determine whether time dimensions match to the window granularity or not. - * @param {PreAggregationReferences} references - * @returns {boolean} */ - const windowGranularityMatches = (references) => { + const windowGranularityMatches = (references: PreAggregationReferences): boolean => { if (!transformedQuery.windowGranularity) { return true; } @@ -708,10 +703,8 @@ export class PreAggregations { /** * Returns an array of 2-element arrays with dimension and granularity. - * @param {*} timeDimension - * @returns {Array>} */ - const expandTimeDimension = (timeDimension) => { + const expandTimeDimension = (timeDimension: string[]): string[][] => { const [dimension, resolvedGranularity] = timeDimension; if (!resolvedGranularity) { return [[dimension, '*']]; // Any granularity should fit From 859c8b345c5cb30a61386ee235d78e7253c72445 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 21 May 2025 16:44:54 +0300 Subject: [PATCH 04/24] fix duplicates in join hint collection --- packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 3542a08be6534..9bbcd4b73b0fa 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -960,7 +960,7 @@ export class CubeSymbols { }; } if (cube[propertyName]) { - return this.cubeReferenceProxy(cubeName, joinHints, propertyName); + return this.cubeReferenceProxy(cubeName, joinHints?.slice(0, -1), propertyName); } if (self.symbols[propertyName]) { return this.cubeReferenceProxy(propertyName, joinHints); From 63f9d0729e53fc58747fbd1ee7005b50e13b1173 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 21 May 2025 16:51:11 +0300 Subject: [PATCH 05/24] Don't even try to match pre-agg if there are customSubQueryJoins --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index b3ad04d47fde0..43cbf37c7113e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -327,6 +327,9 @@ export class BaseQuery { }).filter(R.identity).map(this.newTimeDimension.bind(this)); this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters); /** + * For now this might come only from SQL API, it might be some queries that uses measures and filters to + * get the dimensions that are then used as join conditions to get the final results. + * As consequence - if there are such sub query joins - pre-aggregations can't be used. * @type {Array<{sql: string, on: {expression: Function}, joinType: 'LEFT' | 'INNER', alias: string}>} */ this.customSubQueryJoins = this.options.subqueryJoins ?? []; @@ -703,7 +706,7 @@ export class BaseQuery { if (this.from) { return this.simpleQuery(); } - if (!this.options.preAggregationQuery) { + if (!this.options.preAggregationQuery && !this.customSubQueryJoins.length) { preAggForQuery = this.preAggregations.findPreAggregationForQuery(); if (this.options.disableExternalPreAggregations && preAggForQuery?.preAggregation.external) { From 6c15fb408dc54f7b9a3fb45f972401c354f37a9f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 21 May 2025 17:00:52 +0300 Subject: [PATCH 06/24] Don't try to match pre-aggs if there are MemberExpressions --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 43cbf37c7113e..c580c1579ee0a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -706,7 +706,9 @@ export class BaseQuery { if (this.from) { return this.simpleQuery(); } - if (!this.options.preAggregationQuery && !this.customSubQueryJoins.length) { + const hasMemberExpressions = this.allMembersConcat(false).some(m => m.isMemberExpression); + + if (!this.options.preAggregationQuery && !this.customSubQueryJoins.length && !hasMemberExpressions) { preAggForQuery = this.preAggregations.findPreAggregationForQuery(); if (this.options.disableExternalPreAggregations && preAggForQuery?.preAggregation.external) { From e5f76ecced06ff0133e8ed66c76215071b701128 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 22 May 2025 12:06:04 +0300 Subject: [PATCH 07/24] add joinTree evaluation for pre-agg --- .../src/adapter/PreAggregations.ts | 4 +++- .../src/compiler/CubeEvaluator.ts | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 4effc3935227a..0b41b7d6cacff 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -4,7 +4,8 @@ import { CubeSymbols } from '../compiler/CubeSymbols'; import { UserError } from '../compiler/UserError'; import { BaseQuery } from './BaseQuery'; import { - PreAggregationDefinition, PreAggregationDefinitions, + PreAggregationDefinition, + PreAggregationDefinitions, PreAggregationReferences, PreAggregationTimeDimensionReference } from '../compiler/CubeEvaluator'; @@ -1297,6 +1298,7 @@ export class PreAggregations { const preAggQuery = this.query.preAggregationQueryForSqlEvaluation(cube, aggregation, { inPreAggEvaluation: true }); const aggregateMeasures = preAggQuery?.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }); references.multipliedMeasures = aggregateMeasures?.multipliedMeasures?.map(m => m.measure); + references.joinTree = preAggQuery?.join; } if (aggregation.type === 'rollupLambda') { if (references.rollups.length > 0) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index d8ee7b6bd19b0..5cc197385ee00 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -115,6 +115,25 @@ export type PreAggregationTimeDimensionReference = { granularity: string, }; +// TODO: Move to JonGraph when it will be ts +export type JoinEdge = { + from: string; + to: string; + originalFrom: string; + originalTo: string; + join: { + relationship: string; // TODO Use an enum from validator + sql: Function, + } +}; + +// TODO: Move to JonGraph when it will be ts +export type JoinTree = { + root: string; + joins: JoinEdge[]; + multiplicationFactor: Record; +}; + /// Strings in `dimensions`, `measures` and `timeDimensions[*].dimension` can contain full join path, not just `cube.member` export type PreAggregationReferences = { allowNonStrictDateRangeMatch?: boolean, @@ -123,6 +142,7 @@ export type PreAggregationReferences = { timeDimensions: Array, rollups: Array, multipliedMeasures?: Array, + joinTree?: JoinTree; }; export type PreAggregationInfo = { From ac40600e417352762c6df0fd534a9632765be1c6 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 27 May 2025 17:12:15 +0300 Subject: [PATCH 08/24] more types and ts fixes --- .../src/adapter/BaseDimension.ts | 7 +++- .../src/adapter/BaseFilter.ts | 4 +-- .../src/adapter/BaseMeasure.ts | 11 +++++-- .../src/adapter/BaseQuery.js | 33 +++++++++++++++++-- .../src/adapter/BaseSegment.ts | 11 +++++-- .../src/adapter/PreAggregations.ts | 2 +- .../src/compiler/CubeEvaluator.ts | 12 ++++--- .../src/compiler/CubeToMetaTransformer.js | 2 +- 8 files changed, 63 insertions(+), 19 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts index 34e8fde294ed4..0009b07cbaade 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts @@ -142,6 +142,11 @@ export class BaseDimension { if (this.expression) { return `expr:${this.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); + const path = this.path(); + if (path === null) { + // Sanity check, this should not actually happen because we checked this.expression earlier + throw new Error('Unexpected null path'); + } + return this.query.cubeEvaluator.pathFromArray(path); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts index f656b3249affd..f4fea50f91cb2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts @@ -1,6 +1,6 @@ import inlection from 'inflection'; import moment from 'moment-timezone'; -import { contains, join, map } from 'ramda'; +import { includes, join, map } from 'ramda'; import { FROM_PARTITION_RANGE, TO_PARTITION_RANGE } from '@cubejs-backend/shared'; import { BaseDimension } from './BaseDimension'; @@ -134,7 +134,7 @@ export class BaseFilter extends BaseDimension { } public isDateOperator(): boolean { - return contains(this.camelizeOperator, DATE_OPERATORS); + return includes(this.camelizeOperator, DATE_OPERATORS); } public valuesArray() { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts index dff05deb9ff5f..2acb3bf19c0f2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts @@ -320,17 +320,22 @@ export class BaseMeasure { return this.measureDefinition().sql; } - public path() { + public path(): Array | null { if (this.expression) { return null; } return this.query.cubeEvaluator.parsePath('measures', this.measure); } - public expressionPath() { + public expressionPath(): string { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); + const path = this.path(); + if (path === null) { + // Sanity check, this should not actually happen because we checked this.expression earlier + throw new Error('Unexpected null path'); + } + return this.query.cubeEvaluator.pathFromArray(path); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c580c1579ee0a..32e71e37b8618 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -418,7 +418,7 @@ export class BaseQuery { /** * - * @returns {Array>} + * @returns {Array>} */ get allJoinHints() { if (!this.collectedJoinHints) { @@ -2136,6 +2136,12 @@ export class BaseQuery { )); } + /** + * + * @param {string} cube + * @param {boolean} [isLeftJoinCondition] + * @returns {[string, string, string?]} + */ rewriteInlineCubeSql(cube, isLeftJoinCondition) { const sql = this.cubeSql(cube); const cubeAlias = this.cubeAlias(cube); @@ -2243,6 +2249,11 @@ export class BaseQuery { return this.filtersWithoutSubQueriesValue; } + /** + * + * @param {string} dimension + * @returns {{ prefix: string, subQuery: this, cubeName: string }} + */ subQueryDescription(dimension) { const symbol = this.cubeEvaluator.dimensionByPath(dimension); const [cubeName, name] = this.cubeEvaluator.parsePath('dimensions', dimension); @@ -2287,6 +2298,12 @@ export class BaseQuery { return { prefix, subQuery, cubeName }; } + /** + * + * @param {string} cubeName + * @param {string} name + * @returns {string} + */ subQueryName(cubeName, name) { return `${cubeName}_${name}_subquery`; } @@ -2647,6 +2664,11 @@ export class BaseQuery { ); } + /** + * + * @param {() => void} fn + * @returns {Array} + */ collectSubQueryDimensionsFor(fn) { const context = { subQueryDimensions: [] }; this.evaluateSymbolSqlWithContext( @@ -3209,6 +3231,11 @@ export class BaseQuery { return strings.join(' || '); } + /** + * + * @param {string} cubeName + * @returns {Array} + */ primaryKeyNames(cubeName) { const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName]; if (!primaryKeys || !primaryKeys.length) { @@ -3715,8 +3742,8 @@ export class BaseQuery { /** * - * @param options - * @returns {BaseQuery} + * @param {unknown} options + * @returns {this} */ newSubQuery(options) { const QueryClass = this.constructor; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts index 854eb8da1c8f2..88b626c72b3c6 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts @@ -81,17 +81,22 @@ export class BaseSegment { return this.segmentDefinition().sql; } - public path() { + public path(): Array | null { if (this.expression) { return null; } return this.query.cubeEvaluator.parsePath('segments', this.segment); } - public expressionPath() { + public expressionPath(): string { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); + const path = this.path(); + if (path === null) { + // Sanity check, this should not actually happen because we checked this.expression earlier + throw new Error('Unexpected null path'); + } + return this.query.cubeEvaluator.pathFromArray(path); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0b41b7d6cacff..0bc8e3ee9710e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -758,7 +758,7 @@ export class PreAggregations { if (td[1] === '*') { return R.any((tdtc: [string, string]) => tdtc[0] === td[0]); // need to match the dimension at least } else { - return R.contains(td); + return R.includes(td); } })) ) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 5cc197385ee00..0633853dc3435 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -3,13 +3,15 @@ import R from 'ramda'; import { CubeSymbols, type ToString } from './CubeSymbols'; import { UserError } from './UserError'; -import { BaseQuery } from '../adapter'; +import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; +// TODO replace Function with proper types + export type SegmentDefinition = { type: string, - sql: Function, + sql(): string, primaryKey?: true, ownedByCube: boolean, fieldType?: string, @@ -19,7 +21,7 @@ export type SegmentDefinition = { export type DimensionDefinition = { type: string, - sql: Function, + sql(): string, primaryKey?: true, ownedByCube: boolean, fieldType?: string, @@ -41,7 +43,7 @@ export type TimeShiftDefinitionReference = { export type MeasureDefinition = { type: string, - sql: Function, + sql(): string, ownedByCube: boolean, rollingWindow?: any filters?: any @@ -684,7 +686,7 @@ export class CubeEvaluator extends CubeSymbols { return this.byPath('segments', segmentPath); } - public cubeExists(cube) { + public cubeExists(cube: string): boolean { return !!this.evaluatedCubes[cube]; } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js index fe79d0e62162c..6ee24606aac11 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js @@ -136,7 +136,7 @@ export class CubeToMetaTransformer { // As for now context works on the cubes level return R.filter( - (query) => R.contains(query.config.name, context.contextMembers) + (query) => R.includes(query.config.name, context.contextMembers) )(this.queries); } From a2d984d55368561fb90aab1d671e7348705de29d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 27 May 2025 19:38:08 +0300 Subject: [PATCH 09/24] implement join tree comparison --- .../src/adapter/PreAggregations.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0bc8e3ee9710e..1b93b507d784a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -4,6 +4,7 @@ import { CubeSymbols } from '../compiler/CubeSymbols'; import { UserError } from '../compiler/UserError'; import { BaseQuery } from './BaseQuery'; import { + JoinEdge, PreAggregationDefinition, PreAggregationDefinitions, PreAggregationReferences, @@ -1042,6 +1043,22 @@ export class PreAggregations { ); } + private doesQueryAndPreAggJoinTreeMatch(references: PreAggregationReferences): boolean { + const { query } = this; + const preAggTree = references.joinTree || { joins: [] }; + const preAggEdges = new Set(preAggTree.joins.map((e: JoinEdge) => `${e.from}->${e.to}`)); + const queryTree = query.join; + + for (const edge of queryTree.joins) { + const key = `${edge.from}->${edge.to}`; + if (!preAggEdges.has(key)) { + return false; + } + } + + return true; + } + private evaluatedPreAggregationObj( cube: string, preAggregationName: string, @@ -1053,7 +1070,7 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - canUsePreAggregation: canUsePreAggregation(references), + canUsePreAggregation: canUsePreAggregation(references) && this.doesQueryAndPreAggJoinTreeMatch(references), references, preAggregationId: `${cube}.${preAggregationName}` }; From c90ca5d4c503f6138004e18dbeb8fa3ebad6c15c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 27 May 2025 20:05:15 +0300 Subject: [PATCH 10/24] fix tests --- .../test/integration/postgres/pre-aggregations.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index f862859fc62cf..ea3ad7b767507 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -2230,6 +2230,7 @@ describe('PreAggregations', () => { order: [{ id: 'visitors.source', }], + timezone: 'UTC', }); const queryAndParams = query.buildSqlAndParams(); @@ -2269,6 +2270,7 @@ describe('PreAggregations', () => { }, { id: 'cards.visitorId', }], + timezone: 'UTC', }); const queryAndParams = query.buildSqlAndParams(); From 30ad641ec8ff4e38cf91c785a0fce9e1f473963f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 27 May 2025 20:50:49 +0300 Subject: [PATCH 11/24] fix for 'rollupJoin' pre-aggs --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 2 +- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 32e71e37b8618..dab9d7c9b90e3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -783,7 +783,7 @@ export class BaseQuery { externalPreAggregationQuery() { if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery(); - if (preAggregationForQuery && preAggregationForQuery.preAggregation.external) { + if (preAggregationForQuery?.preAggregation.external) { return true; } const preAggregationsDescription = this.preAggregations.preAggregationsDescription(); diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 1b93b507d784a..fbae3831e3492 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1070,7 +1070,8 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - canUsePreAggregation: canUsePreAggregation(references) && this.doesQueryAndPreAggJoinTreeMatch(references), + // There are no connections in the joinTree between cubes from different datasources for 'rollupJoin' pre-aggs + canUsePreAggregation: canUsePreAggregation(references) && (preAggregation.type === 'rollupJoin' || this.doesQueryAndPreAggJoinTreeMatch(references)), references, preAggregationId: `${cube}.${preAggregationName}` }; From c78cbf273d90f884ba525c04833de6bbc7133bab Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 28 May 2025 15:23:06 +0300 Subject: [PATCH 12/24] remove comments --- .../src/adapter/PreAggregations.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index fbae3831e3492..b9a071038ac80 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -577,11 +577,6 @@ export class PreAggregations { * for specified query, or its value for `refs` if specified. */ public static canUsePreAggregationForTransformedQueryFn(transformedQuery: TransformedQuery, refs: PreAggregationReferences | null = null): CanUsePreAggregationFn { - // TODO this needs to check not only members list, but their join paths as well: - // query can have same members as pre-agg, but different calculated join path - // `refs` will come from pre-agg references, and would contain full join paths - - // TODO remove this in favor of matching with join path function trimmedReferences(references: PreAggregationReferences): PreAggregationReferences { const timeDimensionsTrimmed = references .timeDimensions @@ -643,7 +638,6 @@ export class PreAggregations { * Determine whether pre-aggregation can be used or not. */ const canUsePreAggregationNotAdditive: CanUsePreAggregationFn = (references: PreAggregationReferences): boolean => { - // TODO remove this in favor of matching with join path const referencesTrimmed = trimmedReferences(references); const refTimeDimensions = backAlias(sortTimeDimensions(referencesTrimmed.timeDimensions)); @@ -728,7 +722,6 @@ export class PreAggregations { ? transformedQuery.ownedTimeDimensionsAsIs.map(expandTimeDimension) : transformedQuery.ownedTimeDimensionsWithRollupGranularity.map(expandTimeDimension); - // TODO remove this in favor of matching with join path const referencesTrimmed = trimmedReferences(references); // Even if there are no multiplied measures in the query (because no multiplier dimensions are requested) @@ -1257,7 +1250,6 @@ export class PreAggregations { ) && !!references.dimensions.find((d) => { // `d` can contain full join path, so we should trim it - // TODO check full join path match here const trimmedDimension = CubeSymbols.joinHintFromPath(d).path; return this.query.cubeEvaluator.dimensionByPath(trimmedDimension).primaryKey; }), @@ -1306,10 +1298,6 @@ export class PreAggregations { } private evaluateAllReferences(cube: string, aggregation: PreAggregationDefinition, preAggregationName: string | null = null, context: EvaluateReferencesContext = {}): PreAggregationReferences { - // TODO build a join tree for all references, so they would always include full join path - // Even for preaggregation references without join path - // It is necessary to be able to match query and preaggregation based on full join tree - const evaluateReferences = () => { const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); if (!context.inPreAggEvaluation) { From fd2ec7da34b4773e17d42b4b825b0469315f71d4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 12:34:32 +0300 Subject: [PATCH 13/24] fix tests --- .../test/integration/postgres/pre-aggregations.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index ea3ad7b767507..6c47cd0b2592a 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -2250,7 +2250,7 @@ describe('PreAggregations', () => { [ { visitors__source: 'google', vc__count: '1' }, { visitors__source: 'some', vc__count: '5' }, - { visitors__source: null, vc__count: null }, + { visitors__source: null, vc__count: '0' }, ], ); }); @@ -2291,7 +2291,7 @@ describe('PreAggregations', () => { { visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' }, { visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' }, { visitors__source: 'some', cards__visitor_id: null, vc__count: '2' }, - { visitors__source: null, cards__visitor_id: null, vc__count: null }, + { visitors__source: null, cards__visitor_id: null, vc__count: '0' }, ], ); }); From e827ab6169b3b5365f2392b18eac01c130a96f4c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 13:55:04 +0300 Subject: [PATCH 14/24] fix aliasMember set for segments --- .../cubejs-schema-compiler/src/compiler/CubeEvaluator.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 0633853dc3435..c5adbc263e82b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -530,14 +530,15 @@ export class CubeEvaluator extends CubeSymbols { if (member.sql && !member.subQuery) { const funcArgs = this.funcArguments(member.sql); const { cubeReferencesUsed, evaluatedSql, pathReferencesUsed } = this.collectUsedCubeReferences(cube.name, member.sql); - // We won't check for FILTER_PARAMS here as it shouldn't affect ownership and it should obey the same reference rules. + // We won't check for FILTER_PARAMS here as it shouldn't affect ownership, and it should obey the same reference rules. // To affect ownership FILTER_PARAMS can be declared as `${FILTER_PARAMS.Foo.bar.filter(`${Foo.bar}`)}`. // It isn't owned if there are non {CUBE} references if (funcArgs.length > 0 && cubeReferencesUsed.length === 0) { ownedByCube = false; } // Aliases one to one some another member as in case of views - if (!ownedByCube && !member.filters && CubeSymbols.isCalculatedMeasureType(member.type) && pathReferencesUsed.length === 1 && this.pathFromArray(pathReferencesUsed[0]) === evaluatedSql) { + // Note: Segments do not have type set + if (!ownedByCube && !member.filters && (!member.type || CubeSymbols.isCalculatedMeasureType(member.type)) && pathReferencesUsed.length === 1 && this.pathFromArray(pathReferencesUsed[0]) === evaluatedSql) { aliasMember = this.pathFromArray(pathReferencesUsed[0]); } const foreignCubes = cubeReferencesUsed.filter(usedCube => usedCube !== cube.name); From 68b775822c8886c1ca45e999b49d16b493548b62 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 19:27:16 +0300 Subject: [PATCH 15/24] fix expressionPath() --- packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts | 2 +- packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts index 2acb3bf19c0f2..5221b672cb934 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts @@ -329,7 +329,7 @@ export class BaseMeasure { public expressionPath(): string { if (this.expression) { - return `expr:${this.expression.expressionName}`; + return `expr:${this.expressionName}`; } const path = this.path(); if (path === null) { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts index 88b626c72b3c6..78e8b7b0715a5 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts @@ -90,7 +90,7 @@ export class BaseSegment { public expressionPath(): string { if (this.expression) { - return `expr:${this.expression.expressionName}`; + return `expr:${this.expressionName}`; } const path = this.path(); if (path === null) { From ff7087a11299214f84dfa287d3fa7de1e4cd86d7 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 19:52:09 +0300 Subject: [PATCH 16/24] fix test --- .../test/integration/postgres/sql-generation.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index 9edd2dcd331b9..cc9cef26f9500 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -2267,6 +2267,10 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'day' + }], segments: [ { // eslint-disable-next-line no-new-func @@ -2290,8 +2294,9 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL const res = await testWithPreAggregation(preAggregationsDescription, query); expect(res).toEqual( - // Empty result set, only segments in query - [{}] + [{ + visitors__created_at_day: '2017-01-02T00:00:00.000Z', + }] ); }); From 2d8656550e83a50b354d41f095e04c1f9457eb68 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 19:57:06 +0300 Subject: [PATCH 17/24] add resolveFullMemberPathFn() in BaseQuery --- .../src/adapter/BaseQuery.js | 94 ++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index dab9d7c9b90e3..140b946982ce4 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -363,6 +363,17 @@ export class BaseQuery { try { // TODO allJoinHints should contain join hints form pre-agg this.join = this.joinGraph.buildJoin(this.allJoinHints); + /** + * @type {Record} + */ + const queryJoinGraph = {}; + for (const { originalFrom, originalTo } of (this.join?.joins || [])) { + if (!queryJoinGraph[originalFrom]) { + queryJoinGraph[originalFrom] = []; + } + queryJoinGraph[originalFrom].push(originalTo); + } + this.joinGraphPaths = queryJoinGraph || {}; } catch (e) { if (this.useNativeSqlPlanner) { // Tesseract doesn't require join to be prebuilt and there's a case where single join can't be built for multi-fact query @@ -4889,7 +4900,10 @@ export class BaseQuery { */ backAliasMembers(members) { const query = this; - return Object.fromEntries(members.flatMap( + + const buildJoinPath = this.buildJoinPathFn(); + + const aliases = Object.fromEntries(members.flatMap( member => { const collectedMembers = query.evaluateSymbolSqlWithContext( () => query.collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor'), @@ -4907,5 +4921,83 @@ export class BaseQuery { .map(d => [query.cubeEvaluator.byPathAnyType(d).aliasMember, memberPath]); } )); + + /** + * @type {Record} + */ + const res = {}; + for (const [original, alias] of Object.entries(aliases)) { + const [cube, field] = original.split('.'); + const path = buildJoinPath(cube); + + const [aliasCube, aliasField] = alias.split('.'); + const aliasPath = aliasCube !== cube ? buildJoinPath(aliasCube) : path; + + if (path) { + res[`${path}.${field}`] = aliasPath ? `${aliasPath}.${aliasField}` : alias; + } + + // Aliases might come from proxied members, in such cases + // we need to map them to originals too + if (aliasPath) { + res[original] = `${aliasPath}.${aliasField}`; + } + } + + return res; + } + + buildJoinPathFn() { + const query = this; + const { root } = this.join || {}; + + return (target) => { + const visited = new Set(); + const path = []; + + /** + * @param {string} node + * @returns {boolean} + */ + function dfs(node) { + if (node === target) { + path.push(node); + return true; + } + + if (visited.has(node)) return false; + visited.add(node); + + const neighbors = query.joinGraphPaths[node] || []; + for (const neighbor of neighbors) { + if (dfs(neighbor)) { + path.unshift(node); + return true; + } + } + + return false; + } + + return dfs(root) ? path.join('.') : null; + }; + } + + resolveFullMemberPathFn() { + const { root: queryJoinRoot } = this.join || {}; + + const buildJoinPath = this.buildJoinPathFn(); + + return (member) => { + const [cube, field] = member.split('.'); + if (!cube || !field) return member; + + if (cube === queryJoinRoot.root) { + return member; + } + + const path = buildJoinPath(cube); + return path ? `${path}.${field}` : member; + }; } } From 4ae7d19833b441225afcad1da5b6820f4969711b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 19:58:45 +0300 Subject: [PATCH 18/24] make real full join name path member comparison for pre-agg matching --- .../src/adapter/PreAggregations.ts | 124 ++++++++---------- .../src/compiler/CubeEvaluator.ts | 6 + 2 files changed, 58 insertions(+), 72 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index b9a071038ac80..90971954ac1b8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -390,15 +390,20 @@ export class PreAggregations { } public static transformQueryToCanUseForm(query: BaseQuery): TransformedQuery { - const flattenDimensionMembers = this.flattenDimensionMembers(query); - const sortedDimensions = this.squashDimensions(flattenDimensionMembers); const allBackAliasMembers = query.allBackAliasMembers(); + const resolveFullMemberPath = query.resolveFullMemberPathFn(); + const flattenDimensionMembers = this.flattenDimensionMembers(query); + const sortedDimensions = this.squashDimensions(flattenDimensionMembers).map(resolveFullMemberPath); const measures: (BaseMeasure | BaseFilter | BaseGroupFilter)[] = [...query.measures, ...query.measureFilters]; - const measurePaths = R.uniq(this.flattenMembers(measures).filter((m): m is BaseMeasure => m instanceof BaseMeasure).map(m => m.expressionPath())); + const measurePaths = (R.uniq( + this.flattenMembers(measures) + .filter((m): m is BaseMeasure => m instanceof BaseMeasure) + .map(m => m.expressionPath()) + )).map(resolveFullMemberPath); const collectLeafMeasures = query.collectLeafMeasures.bind(query); const dimensionsList = query.dimensions.map(dim => dim.expressionPath()); const segmentsList = query.segments.map(s => s.expressionPath()); - const ownedDimensions = PreAggregations.ownedMembers(query, flattenDimensionMembers); + const ownedDimensions = PreAggregations.ownedMembers(query, flattenDimensionMembers).map(resolveFullMemberPath); const ownedTimeDimensions = query.timeDimensions.map(td => { const owned = PreAggregations.ownedMembers(query, [td]); let { dimension } = td; @@ -444,7 +449,8 @@ export class PreAggregations { return leafMeasures; }), R.unnest, - R.uniq + R.uniq, + R.map(resolveFullMemberPath), )(measures); function allValuesEq1(map) { @@ -458,8 +464,10 @@ export class PreAggregations { const sortedTimeDimensions = PreAggregations.sortTimeDimensionsWithRollupGranularity(query.timeDimensions); const timeDimensions = PreAggregations.timeDimensionsAsIs(query.timeDimensions); - const ownedTimeDimensionsWithRollupGranularity = PreAggregations.sortTimeDimensionsWithRollupGranularity(ownedTimeDimensions); - const ownedTimeDimensionsAsIs = PreAggregations.timeDimensionsAsIs(ownedTimeDimensions); + const ownedTimeDimensionsWithRollupGranularity = PreAggregations.sortTimeDimensionsWithRollupGranularity(ownedTimeDimensions) + .map(([d, g]) => [resolveFullMemberPath(d), g]); + const ownedTimeDimensionsAsIs = PreAggregations.timeDimensionsAsIs(ownedTimeDimensions) + .map(([d, g]) => [resolveFullMemberPath(d), g]); const hasNoTimeDimensionsWithoutGranularity = !query.timeDimensions.filter(d => !d.granularity).length; @@ -577,31 +585,6 @@ export class PreAggregations { * for specified query, or its value for `refs` if specified. */ public static canUsePreAggregationForTransformedQueryFn(transformedQuery: TransformedQuery, refs: PreAggregationReferences | null = null): CanUsePreAggregationFn { - function trimmedReferences(references: PreAggregationReferences): PreAggregationReferences { - const timeDimensionsTrimmed = references - .timeDimensions - .map(td => ({ - ...td, - dimension: CubeSymbols.joinHintFromPath(td.dimension).path, - })); - const measuresTrimmed = references - .measures - .map(m => CubeSymbols.joinHintFromPath(m).path); - const dimensionsTrimmed = references - .dimensions - .map(d => CubeSymbols.joinHintFromPath(d).path); - const multipliedMeasuresTrimmed = references - .multipliedMeasures?.map(m => CubeSymbols.joinHintFromPath(m).path) || []; - - return { - ...references, - dimensions: dimensionsTrimmed, - measures: measuresTrimmed, - timeDimensions: timeDimensionsTrimmed, - multipliedMeasures: multipliedMeasuresTrimmed, - }; - } - /** * Returns an array of 2-elements arrays with the dimension and granularity * sorted by the concatenated dimension + granularity key. @@ -638,14 +621,12 @@ export class PreAggregations { * Determine whether pre-aggregation can be used or not. */ const canUsePreAggregationNotAdditive: CanUsePreAggregationFn = (references: PreAggregationReferences): boolean => { - const referencesTrimmed = trimmedReferences(references); - - const refTimeDimensions = backAlias(sortTimeDimensions(referencesTrimmed.timeDimensions)); + const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - const backAliasMeasures = backAlias(referencesTrimmed.measures); - const backAliasDimensions = backAlias(referencesTrimmed.dimensions); + const backAliasMeasures = backAlias(references.measures); + const backAliasDimensions = backAlias(references.dimensions); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -657,7 +638,7 @@ export class PreAggregations { R.equals(transformedQuery.timeDimensions, refTimeDimensions) ) && ( filterDimensionsSingleValueEqual && - referencesTrimmed.dimensions.length === filterDimensionsSingleValueEqual.size && + references.dimensions.length === filterDimensionsSingleValueEqual.size && R.all(d => filterDimensionsSingleValueEqual.has(d), backAliasDimensions) || transformedQuery.allFiltersWithinSelectedDimensions && R.equals(backAliasDimensions, transformedQuery.sortedDimensions) @@ -705,7 +686,8 @@ export class PreAggregations { if (!resolvedGranularity) { return [[dimension, '*']]; // Any granularity should fit } - return expandGranularity(dimension, resolvedGranularity) + const trimmedDim = dimension.split('.').slice(-2).join('.'); + return expandGranularity(trimmedDim, resolvedGranularity) .map((newGranularity) => [dimension, newGranularity]); }; @@ -722,30 +704,36 @@ export class PreAggregations { ? transformedQuery.ownedTimeDimensionsAsIs.map(expandTimeDimension) : transformedQuery.ownedTimeDimensionsWithRollupGranularity.map(expandTimeDimension); - const referencesTrimmed = trimmedReferences(references); - // Even if there are no multiplied measures in the query (because no multiplier dimensions are requested) // but the same measures are multiplied in the pre-aggregation, we can't use pre-aggregation // for such queries. - if (referencesTrimmed.multipliedMeasures) { - const backAliasMultipliedMeasures = backAlias(referencesTrimmed.multipliedMeasures); + if (references.multipliedMeasures) { + const backAliasMultipliedMeasures = backAlias(references.multipliedMeasures); - if (transformedQuery.leafMeasures.some(m => referencesTrimmed.multipliedMeasures?.includes(m)) || + if (transformedQuery.leafMeasures.some(m => references.multipliedMeasures?.includes(m)) || transformedQuery.measures.some(m => backAliasMultipliedMeasures.includes(m)) ) { return false; } } + // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are + // no connections in the joinTree between cubes from different datasources + const dimsToMatch = references.fullNameDimensions.length > 0 ? references.fullNameDimensions : references.dimensions; + const dimensionsMatch = (dimensions, doBackAlias) => R.all( d => ( doBackAlias ? - backAlias(referencesTrimmed.dimensions) : - (referencesTrimmed.dimensions) + backAlias(dimsToMatch) : + (dimsToMatch) ).indexOf(d) !== -1, dimensions ); + // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are + // no connections in the joinTree between cubes from different datasources + const timeDimsToMatch = references.fullNameTimeDimensions.length > 0 ? references.fullNameTimeDimensions : references.timeDimensions; + const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( tds => R.anyPass(tds.map((td: [string, string]) => { @@ -758,15 +746,15 @@ export class PreAggregations { ) )( doBackAlias ? - backAlias(sortTimeDimensions(referencesTrimmed.timeDimensions)) : - (sortTimeDimensions(referencesTrimmed.timeDimensions)) + backAlias(sortTimeDimensions(timeDimsToMatch)) : + (sortTimeDimensions(timeDimsToMatch)) ); if (transformedQuery.ungrouped) { const allReferenceCubes = R.pipe(R.map((name: string) => name?.split('.')[0]), R.uniq, R.sortBy(R.identity))([ - ...referencesTrimmed.measures, - ...referencesTrimmed.dimensions, - ...referencesTrimmed.timeDimensions.map(td => td.dimension), + ...references.measures.map(m => CubeSymbols.joinHintFromPath(m).path), + ...references.dimensions.map(d => CubeSymbols.joinHintFromPath(d).path), + ...references.timeDimensions.map(td => CubeSymbols.joinHintFromPath(td.dimension).path), ]); if ( !R.equals(transformedQuery.sortedAllCubeNames, allReferenceCubes) || @@ -778,12 +766,12 @@ export class PreAggregations { } } - const backAliasMeasures = backAlias(referencesTrimmed.measures); + const backAliasMeasures = backAlias(references.measures); return (( windowGranularityMatches(references) ) && ( R.all( - (m: string) => referencesTrimmed.measures.indexOf(m) !== -1, + (m: string) => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures, ) || R.all( m => backAliasMeasures.indexOf(m) !== -1, @@ -1036,22 +1024,6 @@ export class PreAggregations { ); } - private doesQueryAndPreAggJoinTreeMatch(references: PreAggregationReferences): boolean { - const { query } = this; - const preAggTree = references.joinTree || { joins: [] }; - const preAggEdges = new Set(preAggTree.joins.map((e: JoinEdge) => `${e.from}->${e.to}`)); - const queryTree = query.join; - - for (const edge of queryTree.joins) { - const key = `${edge.from}->${edge.to}`; - if (!preAggEdges.has(key)) { - return false; - } - } - - return true; - } - private evaluatedPreAggregationObj( cube: string, preAggregationName: string, @@ -1063,8 +1035,7 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - // There are no connections in the joinTree between cubes from different datasources for 'rollupJoin' pre-aggs - canUsePreAggregation: canUsePreAggregation(references) && (preAggregation.type === 'rollupJoin' || this.doesQueryAndPreAggJoinTreeMatch(references)), + canUsePreAggregation: canUsePreAggregation(references), references, preAggregationId: `${cube}.${preAggregationName}` }; @@ -1304,7 +1275,16 @@ export class PreAggregations { const preAggQuery = this.query.preAggregationQueryForSqlEvaluation(cube, aggregation, { inPreAggEvaluation: true }); const aggregateMeasures = preAggQuery?.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }); references.multipliedMeasures = aggregateMeasures?.multipliedMeasures?.map(m => m.measure); - references.joinTree = preAggQuery?.join; + if (preAggQuery) { + references.joinTree = preAggQuery.join; + const root = references.joinTree?.root || ''; + references.fullNameMeasures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); + references.fullNameDimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); + references.fullNameTimeDimensions = references.timeDimensions.map(d => ({ + dimension: (d.dimension.startsWith(root) ? d.dimension : `${root}.${d.dimension}`), + granularity: d.granularity, + })); + } } if (aggregation.type === 'rollupLambda') { if (references.rollups.length > 0) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index c5adbc263e82b..038fad8c0f98b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -140,8 +140,11 @@ export type JoinTree = { export type PreAggregationReferences = { allowNonStrictDateRangeMatch?: boolean, dimensions: Array, + fullNameDimensions: Array, measures: Array, + fullNameMeasures: Array, timeDimensions: Array, + fullNameTimeDimensions: Array, rollups: Array, multipliedMeasures?: Array, joinTree?: JoinTree; @@ -838,6 +841,9 @@ export class CubeEvaluator extends CubeSymbols { timeDimensions, rollups: aggregation.rollupReferences && this.evaluateReferences(cube, aggregation.rollupReferences, { originalSorting: true }) || [], + fullNameDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() + fullNameMeasures: [], // May be filled in PreAggregations.evaluateAllReferences() + fullNameTimeDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() }; } } From 182690e564bace9f64563121a1a266c567066e94 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 19:58:52 +0300 Subject: [PATCH 19/24] fix tests --- .../test/unit/pre-agg-by-filter-match.test.ts | 3 +++ .../test/unit/pre-agg-time-dim-match.test.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts index 9453b10fad02a..004fb1de3c4f4 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts @@ -59,6 +59,9 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], + fullNameDimensions: [], + fullNameMeasures: [], + fullNameTimeDimensions: [], }; await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index a72aa7df8c943..bea487909c743 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -69,6 +69,9 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], + fullNameDimensions: [], + fullNameMeasures: [], + fullNameTimeDimensions: [], }; await compiler.compile(); From c88ed03baca226e9cf085abf1d54f295e74d3251 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 12 Jun 2025 22:13:04 +0300 Subject: [PATCH 20/24] fix tests --- packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 3a4366b37672a..1ba8b3622a166 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -665,6 +665,9 @@ describe('Refresh Scheduler', () => { measures: ['Foo.count'], timeDimensions: [{ dimension: 'Foo.time', granularity: 'hour' }], rollups: [], + fullNameDimensions: [], + fullNameMeasures: [], + fullNameTimeDimensions: [], }, refreshKey: { every: '1 hour', updateWindow: '1 day', incremental: true }, }, From 75f203e84029892d078bc456249380be085dab71 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 13 Jun 2025 12:04:43 +0300 Subject: [PATCH 21/24] add comments --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 5 +++++ .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 140b946982ce4..5bd8746160be3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -4983,6 +4983,11 @@ export class BaseQuery { }; } + /** + * Returns a function that constructs the full member path + * based on the query's join structure. + * @returns {(function(member: string): (string))} + */ resolveFullMemberPathFn() { const { root: queryJoinRoot } = this.join || {}; diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 90971954ac1b8..048e927a9df96 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1276,6 +1276,12 @@ export class PreAggregations { const aggregateMeasures = preAggQuery?.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }); references.multipliedMeasures = aggregateMeasures?.multipliedMeasures?.map(m => m.measure); if (preAggQuery) { + // We need to build a join tree for all references, so they would always include full join path + // even for preaggregation references without join path. It is necessary to be able to match + // query and preaggregation based on full join tree. But we can not update + // references.{dimensions,measures,timeDimensions} directly, because it will break + // evaluation of references in the query on later stages. + // So we store full named members separately and use them in canUsePreAggregation functions. references.joinTree = preAggQuery.join; const root = references.joinTree?.root || ''; references.fullNameMeasures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); From 235cd98740e823519a5bdb4abf79c955244b83d7 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 13 Jun 2025 13:38:40 +0300 Subject: [PATCH 22/24] Add more tests --- .../postgres/pre-aggregations.test.ts | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 6c47cd0b2592a..499ffb6c237ce 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -1211,7 +1211,7 @@ describe('PreAggregations', () => { }); }); - it('non-match because of join tree difference', async () => { + it('non-match because of join tree difference (through the view)', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ @@ -1269,6 +1269,33 @@ describe('PreAggregations', () => { }); }); + it('non-match because of requesting only joined cube members', async () => { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['visitor_checkins.source'], + order: [{ + id: 'visitor_checkins.source' + }], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect((query).preAggregations.preAggregationForQuery).toBeUndefined(); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual([ + { + vc__source: 'google', + }, + { + vc__source: null, + }, + ]); + }); + }); + it('non-leaf additive measure', async () => { await compiler.compile(); From 9255f51bc25ed849772292de553095d74cdb7d58 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 26 Jun 2025 19:46:08 +0300 Subject: [PATCH 23/24] fix tesseract skip tests --- .../postgres/pre-aggregations.test.ts | 162 ++++++++++-------- 1 file changed, 87 insertions(+), 75 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 499ffb6c237ce..2fc1138e80369 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -1211,90 +1211,102 @@ describe('PreAggregations', () => { }); }); - it('non-match because of join tree difference (through the view)', async () => { - await compiler.compile(); - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - measures: [ - 'cards_visitors_checkins_view.count' - ], - dimensions: ['cards_visitors_checkins_view.source'], - timeDimensions: [{ - dimension: 'cards_visitors_checkins_view.createdAt', - granularity: 'day', - dateRange: ['2017-01-01', '2017-01-30'] - }], - order: [{ - id: 'cards_visitors_checkins_view.createdAt' - }, { - id: 'cards_visitors_checkins_view.source' - }], - timezone: 'America/Los_Angeles', - preAggregationsSchema: '' + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): non-match because of join tree difference (through the view)', () => { + // This should be fixed in Tesseract. }); + } else { + it('non-match because of join tree difference (through the view)', async () => { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'cards_visitors_checkins_view.count' + ], + dimensions: ['cards_visitors_checkins_view.source'], + timeDimensions: [{ + dimension: 'cards_visitors_checkins_view.createdAt', + granularity: 'day', + dateRange: ['2017-01-01', '2017-01-30'] + }], + order: [{ + id: 'cards_visitors_checkins_view.createdAt' + }, { + id: 'cards_visitors_checkins_view.source' + }], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - expect((query).preAggregations.preAggregationForQuery).toBeUndefined(); + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect((query).preAggregations.preAggregationForQuery).toBeUndefined(); - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - expect(res).toEqual( - [ - { - cards_visitors_checkins_view__count: '1', - cards_visitors_checkins_view__created_at_day: '2017-01-02T00:00:00.000Z', - cards_visitors_checkins_view__source: 'google', - }, - { - cards_visitors_checkins_view__count: '1', - cards_visitors_checkins_view__created_at_day: '2017-01-02T00:00:00.000Z', - cards_visitors_checkins_view__source: null, - }, - { - cards_visitors_checkins_view__count: '1', - cards_visitors_checkins_view__created_at_day: '2017-01-04T00:00:00.000Z', - cards_visitors_checkins_view__source: null, - }, - { - cards_visitors_checkins_view__count: '1', - cards_visitors_checkins_view__created_at_day: '2017-01-05T00:00:00.000Z', - cards_visitors_checkins_view__source: null, - }, - { - cards_visitors_checkins_view__count: '2', - cards_visitors_checkins_view__created_at_day: '2017-01-06T00:00:00.000Z', - cards_visitors_checkins_view__source: null, - }, - ] - ); + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [ + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-02T00:00:00.000Z', + cards_visitors_checkins_view__source: 'google', + }, + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-02T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-04T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + { + cards_visitors_checkins_view__count: '1', + cards_visitors_checkins_view__created_at_day: '2017-01-05T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + { + cards_visitors_checkins_view__count: '2', + cards_visitors_checkins_view__created_at_day: '2017-01-06T00:00:00.000Z', + cards_visitors_checkins_view__source: null, + }, + ] + ); + }); }); - }); + } - it('non-match because of requesting only joined cube members', async () => { - await compiler.compile(); - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - dimensions: ['visitor_checkins.source'], - order: [{ - id: 'visitor_checkins.source' - }], - timezone: 'America/Los_Angeles', - preAggregationsSchema: '' + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): non-match because of requesting only joined cube members', () => { + // This should be fixed in Tesseract. }); + } else { + it('non-match because of requesting only joined cube members', async () => { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['visitor_checkins.source'], + order: [{ + id: 'visitor_checkins.source' + }], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - expect((query).preAggregations.preAggregationForQuery).toBeUndefined(); + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect((query).preAggregations.preAggregationForQuery).toBeUndefined(); - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - expect(res).toEqual([ - { - vc__source: 'google', - }, - { - vc__source: null, - }, - ]); + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual([ + { + vc__source: 'google', + }, + { + vc__source: null, + }, + ]); + }); }); - }); + } it('non-leaf additive measure', async () => { await compiler.compile(); From 23772781b7fa47d7a5e138fbdd635016dbc19c00 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 27 Jun 2025 13:04:27 +0300 Subject: [PATCH 24/24] mark some tests as skipped for tesseract --- .../postgres/multi-fact-join.test.ts | 44 ++-- .../postgres/pre-aggregations.test.ts | 218 ++++++++++-------- 2 files changed, 143 insertions(+), 119 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts index a2eac754d94b2..d687e383b514e 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts @@ -112,23 +112,29 @@ cube(\`city\`, { ); } - it('two regular sub-queries', async () => runQueryTest({ - measures: ['orders.amount', 'shipments.count'], - dimensions: [ - 'city.name' - ], - order: [{ id: 'city.name' }] - }, [{ - city__name: 'New York City', - orders__amount: '9', - shipments__count: '3', - }, { - city__name: 'San Francisco', - orders__amount: '6', - shipments__count: '1', - }, { - city__name: null, - orders__amount: '6', - shipments__count: '1', - }])); + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): two regular sub-queries', () => { + // TODO: Fix in tesseract + }); + } else { + it('two regular sub-queries', async () => runQueryTest({ + measures: ['orders.amount', 'shipments.count'], + dimensions: [ + 'city.name' + ], + order: [{ id: 'city.name' }] + }, [{ + city__name: 'New York City', + orders__amount: '9', + shipments__count: '3', + }, { + city__name: 'San Francisco', + orders__amount: '6', + shipments__count: '1', + }, { + city__name: null, + orders__amount: '6', + shipments__count: '1', + }])); + } }); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 2fc1138e80369..1d816929a78cd 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -2257,129 +2257,147 @@ describe('PreAggregations', () => { }); }); - it('rollup join', async () => { - await compiler.compile(); - - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - measures: [ - 'visitor_checkins.count', - ], - dimensions: ['visitors.source'], - preAggregationsSchema: '', - order: [{ - id: 'visitors.source', - }], - timezone: 'UTC', + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollup join', () => { + // TODO: Fix in tesseract }); + } else { + it('rollup join', async () => { + await compiler.compile(); - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); - console.log(preAggregationsDescription); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.count', + ], + dimensions: ['visitors.source'], + preAggregationsSchema: '', + order: [{ + id: 'visitors.source', + }], + timezone: 'UTC', + }); - console.log(query.preAggregations?.rollupMatchResultDescriptions()); + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); - const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); + console.log(query.preAggregations?.rollupMatchResultDescriptions()); - console.log(JSON.stringify(queries.concat(queryAndParams))); + const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - console.log(JSON.stringify(res)); - expect(res).toEqual( - [ - { visitors__source: 'google', vc__count: '1' }, - { visitors__source: 'some', vc__count: '5' }, - { visitors__source: null, vc__count: '0' }, - ], - ); - }); - }); + console.log(JSON.stringify(queries.concat(queryAndParams))); - it('rollup join existing joins', async () => { - await compiler.compile(); + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + console.log(JSON.stringify(res)); + expect(res).toEqual( + [ + { visitors__source: 'google', vc__count: '1' }, + { visitors__source: 'some', vc__count: '5' }, + { visitors__source: null, vc__count: '0' }, + ], + ); + }); + }); + } - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - measures: [ - 'visitor_checkins.count', - ], - dimensions: ['visitors.source', 'cards.visitorId'], - preAggregationsSchema: '', - order: [{ - id: 'visitors.source', - }, { - id: 'cards.visitorId', - }], - timezone: 'UTC', + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollup join existing joins', () => { + // TODO: Fix in tesseract }); + } else { + it('rollup join existing joins', async () => { + await compiler.compile(); - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); - console.log(preAggregationsDescription); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.count', + ], + dimensions: ['visitors.source', 'cards.visitorId'], + preAggregationsSchema: '', + order: [{ + id: 'visitors.source', + }, { + id: 'cards.visitorId', + }], + timezone: 'UTC', + }); - console.log(query.preAggregations?.rollupMatchResultDescriptions()); + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); - const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); + console.log(query.preAggregations?.rollupMatchResultDescriptions()); - console.log(JSON.stringify(queries.concat(queryAndParams))); + const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - console.log(JSON.stringify(res)); - expect(res).toEqual( - [ - { visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' }, - { visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' }, - { visitors__source: 'some', cards__visitor_id: null, vc__count: '2' }, - { visitors__source: null, cards__visitor_id: null, vc__count: '0' }, - ], - ); - }); - }); + console.log(JSON.stringify(queries.concat(queryAndParams))); - it('rollup join partitioned', async () => { - await compiler.compile(); + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + console.log(JSON.stringify(res)); + expect(res).toEqual( + [ + { visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' }, + { visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' }, + { visitors__source: 'some', cards__visitor_id: null, vc__count: '2' }, + { visitors__source: null, cards__visitor_id: null, vc__count: '0' }, + ], + ); + }); + }); + } - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - measures: [ - 'visitor_checkins.count', - ], - dimensions: ['visitors.source'], - timezone: 'America/Los_Angeles', - preAggregationsSchema: '', - timeDimensions: [{ - dimension: 'visitors.createdAt', - granularity: 'hour', - dateRange: ['2017-01-03', '2017-01-04'] - }], - order: [{ - id: 'visitors.createdAt' - }], + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollup join partitioned', () => { + // TODO: Fix in tesseract }); + } else { + it('rollup join partitioned', async () => { + await compiler.compile(); - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); - console.log(preAggregationsDescription); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.count', + ], + dimensions: ['visitors.source'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '', + timeDimensions: [{ + dimension: 'visitors.createdAt', + granularity: 'hour', + dateRange: ['2017-01-03', '2017-01-04'] + }], + order: [{ + id: 'visitors.createdAt' + }], + }); - console.log(query.preAggregations?.rollupMatchResultDescriptions()); + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); - const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); + console.log(query.preAggregations?.rollupMatchResultDescriptions()); - console.log(JSON.stringify(queries.concat(queryAndParams))); + const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - console.log(JSON.stringify(res)); - expect(res).toEqual( - [ - { - visitors__source: 'some', - visitors__created_at_hour: '2017-01-04T16:00:00.000Z', - vc__count: '2' - } - ], - ); + console.log(JSON.stringify(queries.concat(queryAndParams))); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + console.log(JSON.stringify(res)); + expect(res).toEqual( + [ + { + visitors__source: 'some', + visitors__created_at_hour: '2017-01-04T16:00:00.000Z', + vc__count: '2' + } + ], + ); + }); }); - }); + } it('partitioned without time', async () => { await compiler.compile();