Skip to content

Commit b075966

Browse files
KSDaemonKSDaemon
authored
fix(schema-compiler): Fix incorrect truncated time dimensions over time series queries for BigQuery (#9615)
* fix(schema-compiler): Fix incorrect truncated time dimensions over time series queries for BigQuery * neat yarn cmd * add drivers tests * update excluded tests * update snapshots * update snapshots * update snapshots * fix DATE tests in BQ * fix snapshot * fix tests * fix tests/snapshots * fix depr windows-2019
1 parent e17baa0 commit b075966

File tree

39 files changed

+36858
-789
lines changed

39 files changed

+36858
-789
lines changed

‎.github/workflows/rust-cubestore.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ jobs:
114114
- x86_64-apple-darwin
115115
- aarch64-apple-darwin
116116
include:
117-
- os: windows-2019
117+
- os: windows-2022
118118
target: x86_64-pc-windows-msvc
119119
executable_name: cubestored.exe
120120
strip: true

‎packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,6 @@ export class BigqueryQuery extends BaseQuery {
142142
return new BigqueryFilter(this, filter);
143143
}
144144

145-
public dateSeriesSql(timeDimension: BaseTimeDimension) {
146-
return `${timeDimension.dateSeriesAliasName()} AS (${this.seriesSql(timeDimension)})`;
147-
}
148-
149145
public seriesSql(timeDimension: BaseTimeDimension) {
150146
const values = timeDimension.timeSeries().map(
151147
([from, to]) => `select '${from}' f, '${to}' t`
@@ -161,26 +157,6 @@ export class BigqueryQuery extends BaseQuery {
161157
return 6;
162158
}
163159

164-
public overTimeSeriesSelect(cumulativeMeasures, dateSeriesSql, baseQuery, dateJoinConditionSql, baseQueryAlias) {
165-
const forSelect = this.overTimeSeriesForSelect(cumulativeMeasures);
166-
const outerSeriesAlias = this.cubeAlias('outer_series');
167-
const outerBase = this.cubeAlias('outer_base');
168-
const timeDimensionAlias = this.timeDimensions.map(d => d.aliasName()).filter(d => !!d)[0];
169-
const aliasesForSelect = this.timeDimensions.map(d => d.dateSeriesSelectColumn(outerSeriesAlias)).concat(
170-
this.dimensions.concat(cumulativeMeasures).map(s => s.aliasName())
171-
).filter(c => !!c).join(', ');
172-
const dateSeriesAlias = this.timeDimensions.map(d => `${d.dateSeriesAliasName()}`).filter(c => !!c)[0];
173-
return `
174-
WITH ${dateSeriesSql} SELECT ${aliasesForSelect} FROM
175-
${dateSeriesAlias} ${outerSeriesAlias}
176-
LEFT JOIN (
177-
SELECT ${forSelect} FROM ${dateSeriesAlias}
178-
INNER JOIN (${baseQuery}) AS ${baseQueryAlias} ON ${dateJoinConditionSql}
179-
${this.groupByClause()}
180-
) AS ${outerBase} ON ${outerSeriesAlias}.${this.escapeColumnName('date_from')} = ${outerBase}.${timeDimensionAlias}
181-
`;
182-
}
183-
184160
public subtractInterval(date, interval) {
185161
const [intervalFormatted, timeUnit] = this.formatInterval(interval);
186162
if (['YEAR', 'MONTH', 'QUARTER'].includes(timeUnit) || intervalFormatted.includes('WEEK')) {
@@ -346,6 +322,7 @@ export class BigqueryQuery extends BaseQuery {
346322
// DATEADD is being rewritten to DATE_ADD
347323
templates.functions.DATE_ADD = 'DATETIME_ADD(DATETIME({{ args[0] }}), INTERVAL {{ interval }} {{ date_part }})';
348324
templates.functions.CURRENTDATE = 'CURRENT_DATE';
325+
templates.functions.DATE = 'TIMESTAMP({{ args_concat }})';
349326
delete templates.functions.TO_CHAR;
350327
delete templates.functions.PERCENTILECONT;
351328
templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}';

‎packages/cubejs-testing-drivers/fixtures/athena.json

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,9 @@
150150
"---------------------------------------",
151151
"querying BigECommerce: rolling window by 2 day without date range",
152152
"querying BigECommerce: rolling window by 2 month without date range",
153-
"querying BigECommerce: rolling window YTD",
154153
"querying BigECommerce: rolling window YTD without date range",
155-
"--------------------",
156154

155+
"--------------------",
157156
"week granularity is not supported for intervals",
158157
"--------------------",
159158
"querying BigECommerce: rolling window by 2 week",
@@ -209,6 +208,11 @@
209208
"SQL API: Nested Rollup over asterisk",
210209
"SQL API: Extended nested Rollup over asterisk",
211210
"SQL API: Timeshift measure from cube",
212-
"SQL API: SQL push down push to cube quoted alias"
211+
"SQL API: SQL push down push to cube quoted alias",
212+
"querying BigECommerce: rolling window YTD (month + week)",
213+
"querying BigECommerce: rolling window YTD (month + week + no gran)",
214+
"querying BigECommerce: rolling window YTD without granularity",
215+
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
216+
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
213217
]
214218
}

‎packages/cubejs-testing-drivers/fixtures/bigquery.json

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -163,8 +163,8 @@
163163
"querying BigECommerce: rolling window by 2 day without date range",
164164
"querying BigECommerce: rolling window by 2 month without date range",
165165
"querying BigECommerce: rolling window YTD without date range",
166-
"--------------------",
167166

167+
"---------------------------------------",
168168
"SKIPPED SQL API (Need work)",
169169
"---------------------------------------",
170170
"SQL API: reuse params",
@@ -198,14 +198,9 @@
198198
"SQL API: Rollup with aliases",
199199
"SQL API: Nested Rollup over asterisk",
200200
"SQL API: Extended nested Rollup over asterisk",
201-
"querying Products: dimensions -- doesn't work wo ordering",
202-
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
203-
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
204-
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
205201
"querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + no dimension",
206202
"querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + dimension",
207203
"pre-aggregations Customers: running total without time dimension",
208-
"querying BigECommerce: null boolean",
209204
"querying BigECommerce: rolling count_distinct_approx window by 2 day",
210205
"querying BigECommerce: rolling count_distinct_approx window by 2 week",
211206
"querying BigECommerce: rolling count_distinct_approx window by 2 month",
@@ -216,6 +211,13 @@
216211
"SQL API: Nested Rollup",
217212
"SQL API: Nested Rollup with aliases",
218213
"SQL API: Timeshift measure from cube",
219-
"SQL API: SQL push down push to cube quoted alias"
214+
215+
"---- Different results comparing to baseQuery version. Need to investigate ----",
216+
"SQL API: SQL push down push to cube quoted alias",
217+
"querying BigECommerce: rolling window YTD (month + week)",
218+
"querying BigECommerce: rolling window YTD (month + week + no gran)",
219+
"querying BigECommerce: rolling window YTD without granularity",
220+
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
221+
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
220222
]
221223
}

‎packages/cubejs-testing-drivers/fixtures/clickhouse.json

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -179,20 +179,25 @@
179179
"querying BigECommerce: partitioned pre-agg",
180180
"querying BigECommerce: null sum",
181181
"querying BigECommerce: null boolean",
182+
183+
"---------------------------------------",
182184
"Unsupported JOIN ON conditions. Unexpected 'big_e_commerce__order_date_month > subtractWeeks(date_to, 2)'",
183-
"--------------------",
185+
"---------------------------------------",
184186
"querying BigECommerce: rolling window by 2 day",
185187
"querying BigECommerce: rolling window by 2 week",
186188
"querying BigECommerce: rolling window by 2 month",
189+
"querying BigECommerce: rolling window YTD (month)",
190+
"querying BigECommerce: rolling window YTD (month + week)",
191+
"querying BigECommerce: rolling window YTD (month + week + no gran)",
192+
"querying BigECommerce: rolling window YTD (month + week + day)",
193+
"querying BigECommerce: rolling window YTD (month + week + day + no gran)",
187194

188195
"---------------------------------------",
189196
"Requires Tesseract. ",
190197
"---------------------------------------",
191198
"querying BigECommerce: rolling window by 2 day without date range",
192199
"querying BigECommerce: rolling window by 2 month without date range",
193-
"querying BigECommerce: rolling window YTD",
194200
"querying BigECommerce: rolling window YTD without date range",
195-
"--------------------",
196201

197202
"---------------------------------------",
198203
"Custom Granularities ",
@@ -210,6 +215,11 @@
210215
"SQL API: Rollup over exprs",
211216
"SQL API: Rollup with aliases",
212217
"SQL API: Simple Rollup",
213-
"SQL API: SQL push down push to cube quoted alias"
218+
"SQL API: SQL push down push to cube quoted alias",
219+
220+
"SKIPPED SQL API (due to inconsistency)",
221+
"---------------------------------------",
222+
"Below doesn't work probably due to strict type comparison in ClickHouse, but tests run across all DBs",
223+
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
214224
]
215225
}

‎packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,9 +215,7 @@
215215
"---------------------------------------",
216216
"querying BigECommerce: rolling window by 2 day without date range",
217217
"querying BigECommerce: rolling window by 2 month without date range",
218-
"querying BigECommerce: rolling window YTD",
219218
"querying BigECommerce: rolling window YTD without date range",
220-
"--------------------",
221219

222220
"---------------------------------------",
223221
"Custom Granularities ",

‎packages/cubejs-testing-drivers/fixtures/mssql.json

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,9 +141,8 @@
141141
"---------------------------------------",
142142
"querying BigECommerce: rolling window by 2 day without date range",
143143
"querying BigECommerce: rolling window by 2 month without date range",
144-
"querying BigECommerce: rolling window YTD",
145144
"querying BigECommerce: rolling window YTD without date range",
146-
"--------------------",
145+
147146
"---------------------------------------",
148147
"SKIPPED SQL API (Need work)",
149148
"---------------------------------------",
@@ -163,6 +162,12 @@
163162
"SQL API: NULLS FIRST/LAST SQL push down",
164163
"SQL API: SQL push down push to cube quoted alias",
165164
"SQL API: Date/time comparison with SQL push down",
166-
"SQL API: Date/time comparison with date_trunc with SQL push down"
165+
"SQL API: Date/time comparison with date_trunc with SQL push down",
166+
167+
"---------------------------------------",
168+
"Error during rewrite: Can't detect Cube query and it may be not supported yet.",
169+
"---------------------------------------",
170+
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
171+
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
167172
]
168173
}

‎packages/cubejs-testing-drivers/fixtures/mysql.json

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,12 +131,12 @@
131131
"querying BigECommerce: partitioned pre-agg",
132132
"querying BigECommerce: null sum",
133133
"querying BigECommerce: null boolean",
134+
134135
"---------------------------------------",
135136
"Requires Tesseract. ",
136137
"---------------------------------------",
137138
"querying BigECommerce: rolling window by 2 day without date range",
138139
"querying BigECommerce: rolling window by 2 month without date range",
139-
"querying BigECommerce: rolling window YTD",
140140
"querying BigECommerce: rolling window YTD without date range",
141141

142142
"---------------------------------------",
@@ -158,6 +158,12 @@
158158
"SQL API: Nested Rollup over asterisk",
159159
"SQL API: Extended nested Rollup over asterisk",
160160
"SQL API: SQL push down push to cube quoted alias",
161-
"SQL API: Date/time comparison with date_trunc with SQL push down"
161+
"SQL API: Date/time comparison with date_trunc with SQL push down",
162+
163+
"---------------------------------------",
164+
"Error during rewrite: Can't detect Cube query and it may be not supported yet.",
165+
"---------------------------------------",
166+
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
167+
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
162168
]
163169
}

‎packages/cubejs-testing-drivers/fixtures/postgres.json

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@
156156
"querying Products: dimensions -- doesn't work wo ordering",
157157
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
158158
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
159+
159160
"---------------------------------------",
160161
"Requires Tesseract. ",
161162
"---------------------------------------",
@@ -181,6 +182,13 @@
181182
"SQL API: Nested Rollup over asterisk",
182183
"SQL API: Extended nested Rollup over asterisk",
183184
"SQL API: Timeshift measure from cube",
184-
"SQL API: SQL push down push to cube quoted alias"
185+
"SQL API: SQL push down push to cube quoted alias",
186+
187+
"---- Different results comparing to baseQuery version. Need to investigate ----",
188+
"querying BigECommerce: rolling window YTD (month + week)",
189+
"querying BigECommerce: rolling window YTD (month + week + no gran)",
190+
"querying BigECommerce: rolling window YTD without granularity",
191+
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
192+
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
185193
]
186194
}

‎packages/cubejs-testing-drivers/fixtures/redshift.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,14 +167,14 @@
167167
"querying Products: dimensions -- doesn't work wo ordering",
168168
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
169169
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
170+
170171
"---------------------------------------",
171172
"Requires Tesseract. ",
172173
"---------------------------------------",
173174
"querying BigECommerce: rolling window by 2 day without date range",
174175
"querying BigECommerce: rolling window by 2 month without date range",
175-
"querying BigECommerce: rolling window YTD",
176176
"querying BigECommerce: rolling window YTD without date range",
177-
177+
178178
"---------------------------------------",
179179
"SKIPPED SQL API (Need work) ",
180180
"---------------------------------------",

0 commit comments

Comments
 (0)