Skip to content

Commit 99fed38

Browse files
cabljaccabljac
authored
feat(bq-change-tracker): materialized views (#2258)
* chore(firestore-bigquery-change-tracker): fix tests * refactor(firestore-bigquery-change-tracker): improve snapshot.ts readability * feat(firestore-bigquery-change-tracker): add initial materialized views to change tracker * fix(firestore-bigquery-change-tracker): address some issues with recreate logic * test(firestore-bigquery-change-tracker): fix tests * refactor(firestore-bigquery-change-tracker): update max_staleness and query * fix(firestore-bigquery-change-tracker): remove debug logs
1 parent 5153673 commit 99fed38

File tree

16 files changed

+1731
-217
lines changed

16 files changed

+1731
-217
lines changed

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/e2e.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ describe("e2e", () => {
126126
datasetId,
127127
tableId,
128128
timePartitioning: "DAY",
129-
timePartitioningField: "created",
129+
timePartitioningField: "timestamp",
130130
timePartitioningFieldType: "TIMESTAMP",
131131
timePartitioningFirestoreField: "created",
132132
}).record([event]);
@@ -141,7 +141,7 @@ describe("e2e", () => {
141141
tableId
142142
);
143143

144-
expect(changeLogRows[0].created.value).toBe(
144+
expect(changeLogRows[0].timestamp.value).toBe(
145145
BigQuery.timestamp(created.toDate()).value
146146
);
147147
});

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestMaterializedView.test.ts

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
import {
2+
BigQuery,
3+
Dataset,
4+
TableMetadata,
5+
Table,
6+
} from "@google-cloud/bigquery";
7+
import { firestore } from "firebase-admin";
8+
import { RawChangelogViewSchema } from "../../../bigquery/schema";
9+
import { initializeLatestMaterializedView } from "../../../bigquery/initializeLatestMaterializedView";
10+
import {
11+
changeTracker,
12+
changeTrackerEvent,
13+
} from "../../fixtures/changeTracker";
14+
import { deleteTable } from "../../fixtures/clearTables";
15+
import * as logs from "../../../logs";
16+
17+
jest.mock("../../../logs");
18+
// jest.mock("sql-formatter");
19+
20+
describe("initializeLatestMaterializedView", () => {
21+
const projectId = "dev-extensions-testing";
22+
const bq = new BigQuery({ projectId });
23+
24+
let dataset: Dataset;
25+
let table: Table;
26+
let testConfig: {
27+
datasetId: string;
28+
tableId: string;
29+
tableIdRaw: string;
30+
viewIdRaw: string;
31+
};
32+
33+
beforeEach(async () => {
34+
const randomId = (Math.random() + 1).toString(36).substring(7);
35+
testConfig = {
36+
datasetId: `dataset_${randomId}`,
37+
tableId: `table_${randomId}`,
38+
tableIdRaw: `table_${randomId}_raw_changelog`,
39+
viewIdRaw: `table_${randomId}_raw_latest`,
40+
};
41+
dataset = bq.dataset(testConfig.datasetId);
42+
table = dataset.table(testConfig.tableIdRaw);
43+
44+
await dataset.create();
45+
46+
await table.create({ schema: RawChangelogViewSchema });
47+
});
48+
49+
afterEach(async () => {
50+
await deleteTable({ datasetId: testConfig.datasetId });
51+
});
52+
53+
test("creates a new materialized view when view does not exist", async () => {
54+
const view = dataset.table(testConfig.viewIdRaw);
55+
const config = {
56+
datasetId: testConfig.datasetId,
57+
tableId: testConfig.tableId,
58+
useMaterializedView: true,
59+
useIncrementalMaterializedView: false,
60+
maxStaleness: `INTERVAL "4:0:0" HOUR TO SECOND`,
61+
refreshIntervalMinutes: 5,
62+
clustering: null,
63+
};
64+
65+
await initializeLatestMaterializedView({
66+
bq,
67+
changeTrackerConfig: config,
68+
view,
69+
viewExists: false,
70+
rawChangeLogTableName: testConfig.tableIdRaw,
71+
rawLatestViewName: testConfig.viewIdRaw,
72+
schema: RawChangelogViewSchema,
73+
});
74+
75+
const [metadata] = (await view.getMetadata()) as unknown as [TableMetadata];
76+
expect(metadata.materializedView).toBeDefined();
77+
expect(metadata.materializedView?.enableRefresh).toBe(true);
78+
expect(
79+
metadata.materializedView?.allowNonIncrementalDefinition
80+
).toBeDefined();
81+
});
82+
83+
test("does not recreate view if configuration matches", async () => {
84+
const event = changeTrackerEvent({
85+
data: { end_date: firestore.Timestamp.now() },
86+
eventId: "testing2",
87+
});
88+
89+
await changeTracker({
90+
datasetId: testConfig.datasetId,
91+
tableId: testConfig.tableId,
92+
useMaterializedView: true,
93+
useIncrementalMaterializedView: true,
94+
}).record([event]);
95+
96+
const view = dataset.table(testConfig.viewIdRaw);
97+
const config = {
98+
datasetId: testConfig.datasetId,
99+
tableId: testConfig.tableId,
100+
useMaterializedView: true,
101+
useIncrementalMaterializedView: true,
102+
clustering: null,
103+
};
104+
105+
const [initialMetadata] = (await view.getMetadata()) as unknown as [
106+
TableMetadata
107+
];
108+
109+
await initializeLatestMaterializedView({
110+
bq,
111+
changeTrackerConfig: config,
112+
view,
113+
viewExists: true,
114+
rawChangeLogTableName: testConfig.tableIdRaw,
115+
rawLatestViewName: testConfig.viewIdRaw,
116+
schema: RawChangelogViewSchema,
117+
});
118+
119+
const [finalMetadata] = (await view.getMetadata()) as unknown as [
120+
TableMetadata
121+
];
122+
expect(finalMetadata).toEqual(initialMetadata);
123+
});
124+
125+
test("recreates view when switching from incremental to non-incremental", async () => {
126+
const event = changeTrackerEvent({
127+
data: { end_date: firestore.Timestamp.now() },
128+
eventId: "testing3",
129+
});
130+
131+
await changeTracker({
132+
datasetId: testConfig.datasetId,
133+
tableId: testConfig.tableId,
134+
useMaterializedView: true,
135+
useIncrementalMaterializedView: true,
136+
}).record([event]);
137+
138+
const view = dataset.table(testConfig.viewIdRaw);
139+
const newConfig = {
140+
datasetId: testConfig.datasetId,
141+
tableId: testConfig.tableId,
142+
useMaterializedView: true,
143+
maxStaleness: `INTERVAL "4:0:0" HOUR TO SECOND`,
144+
refreshIntervalMinutes: 5,
145+
clustering: null,
146+
};
147+
148+
const [initialMetadata] = (await view.getMetadata()) as unknown as [
149+
TableMetadata
150+
];
151+
expect(
152+
initialMetadata.materializedView?.allowNonIncrementalDefinition
153+
).toBeUndefined();
154+
155+
await initializeLatestMaterializedView({
156+
bq,
157+
changeTrackerConfig: newConfig,
158+
view,
159+
viewExists: true,
160+
rawChangeLogTableName: testConfig.tableIdRaw,
161+
rawLatestViewName: testConfig.viewIdRaw,
162+
schema: RawChangelogViewSchema,
163+
});
164+
165+
const [finalMetadata] = (await view.getMetadata()) as unknown as [
166+
TableMetadata
167+
];
168+
expect(
169+
finalMetadata.materializedView?.allowNonIncrementalDefinition
170+
).toBeDefined();
171+
});
172+
173+
test("handles view creation errors", async () => {
174+
const view = dataset.table(testConfig.viewIdRaw);
175+
const invalidConfig = {
176+
datasetId: testConfig.datasetId,
177+
tableId: testConfig.tableId,
178+
useMaterializedView: true,
179+
maxStaleness: "invalid",
180+
clustering: null,
181+
};
182+
183+
await expect(
184+
initializeLatestMaterializedView({
185+
bq,
186+
changeTrackerConfig: invalidConfig,
187+
view,
188+
viewExists: false,
189+
rawChangeLogTableName: testConfig.tableIdRaw,
190+
rawLatestViewName: testConfig.viewIdRaw,
191+
schema: RawChangelogViewSchema,
192+
})
193+
).rejects.toThrow();
194+
195+
expect(logs.tableCreationError).toHaveBeenCalled();
196+
});
197+
});

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestView.test.ts

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import { initializeLatestView } from "../../../bigquery/initializeLatestView";
2+
import { initializeLatestMaterializedView } from "../../../bigquery/initializeLatestMaterializedView";
3+
import { FirestoreBigQueryEventHistoryTrackerConfig } from "../../../bigquery";
4+
5+
jest.mock("../../../bigquery/initializeLatestMaterializedView");
6+
7+
describe("initializeLatestView", () => {
8+
const mockView = {
9+
id: "test_view",
10+
getMetadata: jest.fn(),
11+
setMetadata: jest.fn(),
12+
create: jest.fn(),
13+
};
14+
15+
const mockConfig: FirestoreBigQueryEventHistoryTrackerConfig = {
16+
wildcardIds: true,
17+
datasetId: "test_dataset",
18+
useNewSnapshotQuerySyntax: true,
19+
clustering: [],
20+
tableId: "test_raw_table",
21+
useMaterializedView: false,
22+
};
23+
24+
beforeEach(() => {
25+
jest.clearAllMocks();
26+
});
27+
28+
describe("initializeLatestView", () => {
29+
it("calls initializeLatestMaterializedView when useMaterializedView is true", async () => {
30+
const mockOptions = {
31+
bq: {} as any, // Mocked BigQuery instance
32+
dataset: { id: "test_dataset" } as any, // Mocked Dataset instance
33+
view: mockView as any, // Mocked Table instance
34+
viewExists: false,
35+
rawChangeLogTableName: "test_raw_table",
36+
rawLatestViewName: "test_raw_view",
37+
changeTrackerConfig: { ...mockConfig, useMaterializedView: true },
38+
useMaterializedView: true,
39+
useIncrementalMaterializedView: false,
40+
};
41+
42+
await initializeLatestView(mockOptions);
43+
44+
expect(initializeLatestMaterializedView).toHaveBeenCalled();
45+
});
46+
47+
it("does not call initializeLatestMaterializedView when useMaterializedView is false", async () => {
48+
const mockOptions = {
49+
bq: {} as any, // Mocked BigQuery instance
50+
dataset: { id: "test_dataset" } as any, // Mocked Dataset instance
51+
view: mockView as any, // Mocked Table instance
52+
viewExists: false,
53+
rawChangeLogTableName: "test_raw_table",
54+
rawLatestViewName: "test_raw_view",
55+
changeTrackerConfig: { ...mockConfig, useMaterializedView: false },
56+
useMaterializedView: false,
57+
useIncrementalMaterializedView: false,
58+
};
59+
60+
await initializeLatestView(mockOptions);
61+
62+
expect(initializeLatestMaterializedView).not.toHaveBeenCalled();
63+
});
64+
});
65+
});

0 commit comments

Comments
 (0)