Skip to content

Commit 02f82a1

Browse files
CeridanCeridan
authored
Fix materialized views (#1970)
* Add support for partitionBy and clusterBy fields on materialized views. * Update docs to incl;ude new fields on ViewConfig.
1 parent 08eb041 commit 02f82a1

File tree

5 files changed

+69
-19
lines changed

5 files changed

+69
-19
lines changed

‎core/actions/view.ts

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ import { dataform } from "df/protos/ts";
3434
* Consider breaking backwards compatability of these in v4.
3535
*/
3636
export interface ILegacyViewBigqueryConfig {
37+
partitionBy?: string;
38+
clusterBy?: string[];
3739
labels: { [key: string]: string };
3840
additionalOptions: { [key: string]: string };
3941
}
@@ -193,8 +195,13 @@ export class View extends ActionBuilder<dataform.Table> {
193195
if (config.postOperations) {
194196
this.postOps(config.postOperations);
195197
}
196-
if (Object.keys(config.labels).length || Object.keys(config.additionalOptions).length) {
197-
this.bigquery({ labels: config.labels, additionalOptions: config.additionalOptions });
198+
if (Object.keys(config.labels).length || Object.keys(config.additionalOptions).length || config.partitionBy.length > 0 || config.clusterBy.length > 0) {
199+
this.bigquery({
200+
partitionBy: config.partitionBy,
201+
clusterBy: config.clusterBy,
202+
labels: config.labels,
203+
additionalOptions: config.additionalOptions,
204+
});
198205
}
199206

200207
return this;
@@ -570,6 +577,14 @@ export class View extends ActionBuilder<dataform.Table> {
570577
unverifiedConfig
571578
);
572579
if (unverifiedConfig?.bigquery) {
580+
checkExcessProperties(
581+
(e: Error) => {
582+
throw e;
583+
},
584+
unverifiedConfig.bigquery,
585+
strictKeysOf<ILegacyViewBigqueryConfig>()(["labels", "additionalOptions", "partitionBy", "clusterBy",]),
586+
"BigQuery view config"
587+
);
573588
if (!!unverifiedConfig.bigquery.labels) {
574589
unverifiedConfig.labels = unverifiedConfig.bigquery.labels;
575590
delete unverifiedConfig.bigquery.labels;
@@ -578,22 +593,36 @@ export class View extends ActionBuilder<dataform.Table> {
578593
unverifiedConfig.additionalOptions = unverifiedConfig.bigquery.additionalOptions;
579594
delete unverifiedConfig.bigquery.additionalOptions;
580595
}
581-
checkExcessProperties(
582-
(e: Error) => {
583-
throw e;
584-
},
585-
unverifiedConfig.bigquery,
586-
strictKeysOf<ILegacyViewBigqueryConfig>()(["labels", "additionalOptions"]),
587-
"BigQuery view config"
588-
);
596+
if (!!unverifiedConfig.bigquery.partitionBy) {
597+
unverifiedConfig.partitionBy = unverifiedConfig.bigquery.partitionBy;
598+
delete unverifiedConfig.bigquery.partitionBy;
599+
}
600+
if (!!unverifiedConfig.bigquery.clusterBy) {
601+
unverifiedConfig.clusterBy = unverifiedConfig.bigquery.clusterBy;
602+
delete unverifiedConfig.bigquery.clusterBy;
603+
}
589604
}
590605
}
591606

592-
return verifyObjectMatchesProto(
607+
const config = verifyObjectMatchesProto(
593608
dataform.ActionConfig.ViewConfig,
594609
unverifiedConfig,
595610
VerifyProtoErrorBehaviour.SHOW_DOCS_LINK
596611
);
612+
613+
if (!config.materialized && (config.partitionBy.length > 0 || config.clusterBy.length > 0)) {
614+
this.session.compileError(
615+
`partitionBy/clusterBy can be applied only to materialized views`,
616+
config.filename,
617+
dataform.Target.create({
618+
database: config.project,
619+
schema: config.dataset,
620+
name: config.name
621+
})
622+
);
623+
}
624+
625+
return config;
597626
}
598627
}
599628

‎core/actions/view_test.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ actions:
7777
description: "description",
7878
${exampleActionDescriptor.inputSqlxConfigBlock}
7979
bigquery: {
80+
partitionBy: "partitionBy",
81+
clusterBy: ["clusterBy"],
8082
labels: {"key": "val"},
8183
additionalOptions: {
8284
option1Key: "option1",
@@ -131,6 +133,8 @@ SELECT 1`
131133
disabled: true,
132134
hermeticity: "HERMETIC",
133135
bigquery: {
136+
clusterBy: ["clusterBy"],
137+
partitionBy: "partitionBy",
134138
additionalOptions: {
135139
option1Key: "option1",
136140
option2Key: "option2"
@@ -189,6 +193,9 @@ actions:
189193
- tag2
190194
disabled: true
191195
materialized: true
196+
partitionBy: partitionBy
197+
clusterBy:
198+
- clusterBy
192199
description: description
193200
labels:
194201
key: val
@@ -220,6 +227,8 @@ ${exampleBuiltInAssertionsAsYaml.inputActionConfigBlock}
220227
disabled: true,
221228
hermeticity: "HERMETIC",
222229
bigquery: {
230+
clusterBy: ["clusterBy"],
231+
partitionBy: "partitionBy",
223232
additionalOptions: {
224233
option1Key: "option1",
225234
option2Key: "option2"

‎core/main_test.ts

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1537,28 +1537,30 @@ assert("name", {
15371537
suite("invalid options", () => {
15381538
[
15391539
{
1540-
testName: "partitionBy invalid for BigQuery views",
1540+
testName: "partitionBy invalid for BigQuery non-materialized views",
15411541
fileContents: `
15421542
publish("name", {
15431543
type: "view",
1544+
materialized: false,
15441545
bigquery: {
15451546
partitionBy: "some_partition"
15461547
}
15471548
})`,
15481549
expectedError:
1549-
'Unexpected property "partitionBy" in BigQuery view config. Supported properties are: ["labels","additionalOptions"]'
1550+
'partitionBy/clusterBy can be applied only to materialized views'
15501551
},
15511552
{
1552-
testName: "clusterBy invalid for BigQuery views",
1553+
testName: "clusterBy invalid for BigQuery non-materialized views",
15531554
fileContents: `
15541555
publish("name", {
15551556
type: "view",
1557+
materialized: false,
15561558
bigquery: {
15571559
clusterBy: ["some_cluster"]
15581560
}
15591561
})`,
15601562
expectedError:
1561-
'Unexpected property "clusterBy" in BigQuery view config. Supported properties are: ["labels","additionalOptions"]'
1563+
'partitionBy/clusterBy can be applied only to materialized views'
15621564
},
15631565
{
15641566
testName: "partitionExpirationDays invalid for BigQuery views",
@@ -1570,7 +1572,7 @@ publish("name", {
15701572
}
15711573
})`,
15721574
expectedError:
1573-
'Unexpected property "partitionExpirationDays" in BigQuery view config. Supported properties are: ["labels","additionalOptions"]'
1575+
'Unexpected property "partitionExpirationDays" in BigQuery view config. Supported properties are: ["labels","additionalOptions","partitionBy","clusterBy"]'
15741576
},
15751577
{
15761578
testName: "requirePartitionFilter invalid for BigQuery views",
@@ -1582,7 +1584,7 @@ publish("name", {
15821584
}
15831585
})`,
15841586
expectedError:
1585-
'Unexpected property "requirePartitionFilter" in BigQuery view config. Supported properties are: ["labels","additionalOptions"]'
1587+
'Unexpected property "requirePartitionFilter" in BigQuery view config. Supported properties are: ["labels","additionalOptions","partitionBy","clusterBy"]'
15861588
},
15871589
{
15881590
testName: "partitionExpirationDays invalid for BigQuery materialized views",
@@ -1595,7 +1597,7 @@ publish("name", {
15951597
}
15961598
})`,
15971599
expectedError:
1598-
'Unexpected property "partitionExpirationDays" in BigQuery view config. Supported properties are: ["labels","additionalOptions"]'
1600+
'Unexpected property "partitionExpirationDays" in BigQuery view config. Supported properties are: ["labels","additionalOptions","partitionBy","clusterBy"]'
15991601
},
16001602
{
16011603
testName: "requirePartitionFilter invalid for BigQuery materialized views",
@@ -1608,7 +1610,7 @@ publish("name", {
16081610
}
16091611
})`,
16101612
expectedError:
1611-
'Unexpected property "requirePartitionFilter" in BigQuery view config. Supported properties are: ["labels","additionalOptions"]'
1613+
'Unexpected property "requirePartitionFilter" in BigQuery view config. Supported properties are: ["labels","additionalOptions","partitionBy","clusterBy"]'
16121614
},
16131615
{
16141616
testName: "materialized invalid for BigQuery tables",

‎docs/reference/configs.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,8 @@ Target represents a unique action identifier.
305305
| preOperations | [string](#string) | repeated | Queries to run before `query`. This can be useful for granting permissions. |
306306
| postOperations | [string](#string) | repeated | Queries to run after `query`. |
307307
| materialized | [bool](#bool) | | Applies the materialized view optimization, see https://cloud.google.com/bigquery/docs/materialized-views-intro. |
308+
| partitionBy | [string](#string) | | Optional. Applicable only to materialized view. The key by which to partition the materialized view. Typically the name of a timestamp or the date column. See https://cloud.google.com/bigquery/docs/materialized-views-create#partitioned_materialized_views. |
309+
| clusterBy | [string](#string) | repeated | Optional. Applicable only to materialized view. The keys by which to cluster partitions by. See https://cloud.google.com/bigquery/docs/materialized-views-create#cluster_materialized_views. |
308310
| description | [string](#string) | | Description of the view. |
309311
| columns | [ActionConfig.ColumnDescriptor](#dataform-ActionConfig-ColumnDescriptor) | repeated | Descriptions of columns within the table. |
310312
| labels | [ActionConfig.ViewConfig.LabelsEntry](#dataform-ActionConfig-ViewConfig-LabelsEntry) | repeated | Key-value pairs for BigQuery labels. |

‎protos/configs.proto

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,14 @@ message ActionConfig {
226226
// https://cloud.google.com/bigquery/docs/materialized-views-intro.
227227
bool materialized = 10;
228228

229+
// Optional. Applicable only to materialized view. The key by which to partition the materialized view. Typically the name of a
230+
// timestamp or the date column. See https://cloud.google.com/bigquery/docs/materialized-views-create#partitioned_materialized_views.
231+
string partition_by = 18;
232+
233+
// Optional. Applicable only to materialized view. The keys by which to cluster partitions by. See
234+
// https://cloud.google.com/bigquery/docs/materialized-views-create#cluster_materialized_views.
235+
repeated string cluster_by = 19;
236+
229237
// Description of the view.
230238
string description = 11;
231239

0 commit comments

Comments
 (0)