Skip to content

Commit 0126320

Browse files
AnkitCLIAnkitCLI
committed
debug
1 parent e8cd347 commit 0126320

File tree

2 files changed

+224
-55
lines changed

2 files changed

+224
-55
lines changed

‎src/e2e-test/features/bigquery/multitablesink/BigQueryMultiTableAdditional.feature‎

Lines changed: 160 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
@BigQueryMultiTable_Sink
1616
Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable successful data transfer
1717

18-
@BQ_TWO_SOURCE_BQMT_TEST
18+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_TWO_SINK_BQMT_TEST
1919
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two new tables
2020
Given Open Datafusion Project to configure pipeline
2121
When Expand Plugin group in the LHS plugins list: "Source"
@@ -57,34 +57,162 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
5757
Then Verify the pipeline status is "Succeeded"
5858
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
5959

60-
# @BQ_SINGLE_SOURCE_BQMT_TEST @BQ_SINK_BQMT_TEST
61-
# Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in one table
62-
# Given Open Datafusion Project to configure pipeline
63-
# When Expand Plugin group in the LHS plugins list: "Source"
64-
# When Select plugin: "BigQuery" from the plugins list as: "Source"
65-
# Then Navigate to the properties page of plugin: "BigQuery"
66-
# Then Replace input plugin property: "project" with value: "projectId"
67-
# Then Replace input plugin property: "dataset" with value: "dataset"
68-
# Then Replace input plugin property: "table" with value: "bqSourceTable"
69-
# Then Click on the Get Schema button
70-
# Then Validate "BigQuery" plugin properties
71-
# Then Close the Plugin Properties page
72-
# When Expand Plugin group in the LHS plugins list: "Sink"
73-
# When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
74-
# Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
75-
# Then Navigate to the properties page of plugin: "BigQuery Multi Table"
76-
# And Enter input plugin property: "referenceName" with value: "Reference"
77-
# And Replace input plugin property: "project" with value: "projectId"
78-
# And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
79-
# And Enter input plugin property: "dataset" with value: "dataset"
80-
# Then Override Service account details if set in environment variables
81-
# Then Click plugin property: "flexibleSchema"
82-
# Then Validate "BigQuery Multi Table" plugin properties
83-
# And Close the Plugin Properties page
84-
# Then Save the pipeline
85-
# Then Deploy the pipeline
86-
# Then Run the Pipeline in Runtime
87-
# Then Wait till pipeline is in running state
88-
# Then Open and capture logs
89-
# Then Verify the pipeline status is "Succeeded"
90-
# Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
60+
@BQ_SINGLE_SOURCE_BQMT_TEST @BQ_SINK_BQMT_TEST
61+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in one table
62+
Given Open Datafusion Project to configure pipeline
63+
When Expand Plugin group in the LHS plugins list: "Source"
64+
When Select plugin: "BigQuery" from the plugins list as: "Source"
65+
Then Navigate to the properties page of plugin: "BigQuery"
66+
Then Replace input plugin property: "project" with value: "projectId"
67+
Then Replace input plugin property: "dataset" with value: "dataset"
68+
Then Replace input plugin property: "table" with value: "bqSourceTable"
69+
Then Click on the Get Schema button
70+
Then Validate "BigQuery" plugin properties
71+
Then Close the Plugin Properties page
72+
When Expand Plugin group in the LHS plugins list: "Sink"
73+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
74+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
75+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
76+
And Enter input plugin property: "referenceName" with value: "Reference"
77+
And Replace input plugin property: "project" with value: "projectId"
78+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
79+
And Enter input plugin property: "dataset" with value: "dataset"
80+
Then Override Service account details if set in environment variables
81+
Then Click plugin property: "flexibleSchema"
82+
Then Validate "BigQuery Multi Table" plugin properties
83+
And Close the Plugin Properties page
84+
Then Save the pipeline
85+
Then Deploy the pipeline
86+
Then Run the Pipeline in Runtime
87+
Then Wait till pipeline is in running state
88+
Then Open and capture logs
89+
Then Verify the pipeline status is "Succeeded"
90+
Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
91+
92+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
93+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables
94+
Given Open Datafusion Project to configure pipeline
95+
When Expand Plugin group in the LHS plugins list: "Source"
96+
When Select plugin: "BigQuery" from the plugins list as: "Source"
97+
Then Navigate to the properties page of plugin: "BigQuery"
98+
Then Replace input plugin property: "project" with value: "projectId"
99+
Then Replace input plugin property: "dataset" with value: "dataset"
100+
Then Replace input plugin property: "table" with value: "bqSourceTable"
101+
Then Click on the Get Schema button
102+
Then Validate "BigQuery" plugin properties
103+
Then Close the Plugin Properties page
104+
When Expand Plugin group in the LHS plugins list: "Source"
105+
When Select plugin: "BigQuery" from the plugins list as: "Source"
106+
Then Navigate to the properties page of plugin: "BigQuery2"
107+
Then Replace input plugin property: "project" with value: "projectId"
108+
Then Replace input plugin property: "dataset" with value: "dataset"
109+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
110+
Then Click on the Get Schema button
111+
Then Validate "BigQuery2" plugin properties
112+
Then Close the Plugin Properties page
113+
When Expand Plugin group in the LHS plugins list: "Sink"
114+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
115+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
116+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
117+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
118+
And Enter input plugin property: "referenceName" with value: "Reference"
119+
And Replace input plugin property: "project" with value: "projectId"
120+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
121+
And Enter input plugin property: "dataset" with value: "dataset"
122+
Then Override Service account details if set in environment variables
123+
Then Click plugin property: "flexibleSchema"
124+
Then Validate "BigQuery Multi Table" plugin properties
125+
And Close the Plugin Properties page
126+
Then Save the pipeline
127+
Then Deploy the pipeline
128+
Then Run the Pipeline in Runtime
129+
Then Wait till pipeline is in running state
130+
Then Open and capture logs
131+
Then Verify the pipeline status is "Succeeded"
132+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
133+
134+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
135+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables using truncate
136+
Given Open Datafusion Project to configure pipeline
137+
When Expand Plugin group in the LHS plugins list: "Source"
138+
When Select plugin: "BigQuery" from the plugins list as: "Source"
139+
Then Navigate to the properties page of plugin: "BigQuery"
140+
Then Replace input plugin property: "project" with value: "projectId"
141+
Then Replace input plugin property: "dataset" with value: "dataset"
142+
Then Replace input plugin property: "table" with value: "bqSourceTable"
143+
Then Click on the Get Schema button
144+
Then Validate "BigQuery" plugin properties
145+
Then Close the Plugin Properties page
146+
When Expand Plugin group in the LHS plugins list: "Source"
147+
When Select plugin: "BigQuery" from the plugins list as: "Source"
148+
Then Navigate to the properties page of plugin: "BigQuery2"
149+
Then Replace input plugin property: "project" with value: "projectId"
150+
Then Replace input plugin property: "dataset" with value: "dataset"
151+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
152+
Then Click on the Get Schema button
153+
Then Validate "BigQuery2" plugin properties
154+
Then Close the Plugin Properties page
155+
When Expand Plugin group in the LHS plugins list: "Sink"
156+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
157+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
158+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
159+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
160+
And Enter input plugin property: "referenceName" with value: "Reference"
161+
And Replace input plugin property: "project" with value: "projectId"
162+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
163+
And Enter input plugin property: "dataset" with value: "dataset"
164+
Then Override Service account details if set in environment variables
165+
Then Click plugin property: "flexibleSchema"
166+
Then Toggle BigQuery sink property truncateTable to true
167+
Then Validate "BigQuery Multi Table" plugin properties
168+
And Close the Plugin Properties page
169+
Then Save the pipeline
170+
Then Deploy the pipeline
171+
Then Run the Pipeline in Runtime
172+
Then Wait till pipeline is in running state
173+
Then Open and capture logs
174+
Then Verify the pipeline status is "Succeeded"
175+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
176+
177+
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
178+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables after updating schema
179+
Given Open Datafusion Project to configure pipeline
180+
When Expand Plugin group in the LHS plugins list: "Source"
181+
When Select plugin: "BigQuery" from the plugins list as: "Source"
182+
Then Navigate to the properties page of plugin: "BigQuery"
183+
Then Replace input plugin property: "project" with value: "projectId"
184+
Then Replace input plugin property: "dataset" with value: "dataset"
185+
Then Replace input plugin property: "table" with value: "bqSourceTable"
186+
Then Click on the Get Schema button
187+
Then Validate "BigQuery" plugin properties
188+
Then Close the Plugin Properties page
189+
When Expand Plugin group in the LHS plugins list: "Source"
190+
When Select plugin: "BigQuery" from the plugins list as: "Source"
191+
Then Navigate to the properties page of plugin: "BigQuery2"
192+
Then Replace input plugin property: "project" with value: "projectId"
193+
Then Replace input plugin property: "dataset" with value: "dataset"
194+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
195+
Then Click on the Get Schema button
196+
Then Validate "BigQuery2" plugin properties
197+
Then Close the Plugin Properties page
198+
When Expand Plugin group in the LHS plugins list: "Sink"
199+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
200+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
201+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
202+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
203+
And Enter input plugin property: "referenceName" with value: "Reference"
204+
And Replace input plugin property: "project" with value: "projectId"
205+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
206+
And Enter input plugin property: "dataset" with value: "dataset"
207+
Then Override Service account details if set in environment variables
208+
Then Click plugin property: "flexibleSchema"
209+
Then Select radio button plugin property: "updateSchema" with value: "true"
210+
Then Validate "BigQuery Multi Table" plugin properties
211+
And Close the Plugin Properties page
212+
Then Save the pipeline
213+
Then Deploy the pipeline
214+
Then Run the Pipeline in Runtime
215+
Then Wait till pipeline is in running state
216+
Then Open and capture logs
217+
Then Verify the pipeline status is "Succeeded"
218+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 64 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,8 @@ public static void createTempSourceBQTable() throws IOException, InterruptedExce
298298
"@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST or @BQ_EXISTING_SOURCE_TEST or @BQ_EXISTING_SINK_TEST or " +
299299
"@BQ_EXISTING_SOURCE_DATATYPE_TEST or @BQ_EXISTING_SINK_DATATYPE_TEST or @BQ_UPSERT_SOURCE_TEST or " +
300300
"@BQ_NULL_MODE_SOURCE_TEST or @BQ_UPDATE_SOURCE_DEDUPE_TEST or @BQ_INSERT_INT_SOURCE_TEST or " +
301-
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST")
301+
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST or" +
302+
" BQ_SINGLE_SOURCE_BQMT_TEST")
302303
public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
303304
BigQueryClient.dropBqQuery(bqSourceTable);
304305
PluginPropertyUtils.removePluginProp("bqSourceTable");
@@ -1554,7 +1555,7 @@ public static void emptyExistingBigTableInstanceAndTableName() {
15541555
bigtableExistingTargetTable = StringUtils.EMPTY;
15551556
}
15561557

1557-
@Before(order = 1, value = "@BQ_EXISTING_TARGET_TEST")
1558+
@Before(order = 2, value = "@BQ_EXISTING_TARGET_TEST")
15581559
public static void createSinkTables() throws IOException, InterruptedException {
15591560
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
15601561
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
@@ -1604,35 +1605,32 @@ public static void createSourceTables() throws IOException, InterruptedException
16041605
PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2);
16051606
}
16061607

1607-
@After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1608-
public static void deleteAllBqTables() throws IOException, InterruptedException {
1609-
BigQueryClient.dropBqQuery(bqSourceTable);
1610-
BigQueryClient.dropBqQuery(bqSourceTable2);
1611-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1612-
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1613-
BigQueryClient.dropBqQuery(bqTargetTable);
1614-
BigQueryClient.dropBqQuery(bqTargetTable2);
1615-
PluginPropertyUtils.removePluginProp("bqSourceTable");
1616-
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1617-
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
1618-
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1619-
BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1620-
BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
1621-
}
1622-
1623-
@After(order = 1, value = "@BQ_SINK_BQMT_TEST")
1608+
// @After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1609+
// public static void deleteAllBqTables() throws IOException, InterruptedException {
1610+
// BigQueryClient.dropBqQuery(bqSourceTable);
1611+
// BigQueryClient.dropBqQuery(bqSourceTable2);
1612+
// bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1613+
// bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1614+
// BigQueryClient.dropBqQuery(bqTargetTable);
1615+
// BigQueryClient.dropBqQuery(bqTargetTable2);
1616+
// PluginPropertyUtils.removePluginProp("bqSourceTable");
1617+
// PluginPropertyUtils.removePluginProp("bqSourceTable2");
1618+
// BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
1619+
// BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1620+
// BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1621+
// BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
1622+
// }
1623+
1624+
@After(order = 2, value = "@BQ_SINK_BQMT_TEST")
16241625
public static void deleteTargetBqmtTable() throws IOException, InterruptedException {
16251626
try {
1626-
bqTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTableTabB");
1627+
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
16271628
BigQueryClient.dropBqQuery(bqTargetTable);
1628-
BigQueryClient.dropBqQuery(bqSourceTable);
16291629
BeforeActions.scenario.write("BQ Target table - " + bqTargetTable + " deleted successfully");
1630-
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " deleted successfully");
16311630
bqTargetTable = StringUtils.EMPTY;
16321631
} catch (BigQueryException e) {
16331632
if (e.getMessage().contains("Not found: Table")) {
16341633
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " does not exist");
1635-
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " does not exist");
16361634
} else {
16371635
Assert.fail(e.getMessage());
16381636
}
@@ -1646,4 +1644,47 @@ public static void deleteTempSource2BQTable() throws IOException, InterruptedExc
16461644
PluginPropertyUtils.removePluginProp("bqSourceTable2");
16471645
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
16481646
}
1647+
1648+
@After(order = 2, value = "@BQ_TWO_SOURCE_BQMT_TEST or @BQ_SOURCE_UPDATE_TEST")
1649+
public static void deleteTwoSourceTables() throws IOException, InterruptedException {
1650+
try {
1651+
BigQueryClient.dropBqQuery(bqSourceTable);
1652+
BigQueryClient.dropBqQuery(bqSourceTable2);
1653+
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " deleted successfully");
1654+
BeforeActions.scenario.write("BQ Source table2 - " + bqSourceTable2 + " deleted successfully");
1655+
PluginPropertyUtils.removePluginProp("bqSourceTable");
1656+
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1657+
1658+
} catch (BigQueryException e) {
1659+
if (e.getMessage().contains("Not found: Table")) {
1660+
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " does not exist");
1661+
BeforeActions.scenario.write("BQ Source Table2 " + bqSourceTable2 + " does not exist");
1662+
} else {
1663+
Assert.fail(e.getMessage());
1664+
}
1665+
}
1666+
}
1667+
1668+
@After(order = 2, value = "@BQ_TWO_SINK_BQMT_TEST")
1669+
public static void deleteTwoTargetTables() throws IOException, InterruptedException {
1670+
try {
1671+
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1672+
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1673+
BigQueryClient.dropBqQuery(bqTargetTable);
1674+
BigQueryClient.dropBqQuery(bqTargetTable2);
1675+
BeforeActions.scenario.write("BQ Target table - " + bqTargetTable + " deleted successfully");
1676+
BeforeActions.scenario.write("BQ Target table2 - " + bqTargetTable2 + " deleted successfully");
1677+
bqTargetTable = StringUtils.EMPTY;
1678+
bqTargetTable2 = StringUtils.EMPTY;
1679+
1680+
} catch (BigQueryException e) {
1681+
if (e.getMessage().contains("Not found: Table")) {
1682+
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " does not exist");
1683+
BeforeActions.scenario.write("BQ Target Table2 " + bqTargetTable2 + " does not exist");
1684+
} else {
1685+
Assert.fail(e.getMessage());
1686+
}
1687+
}
1688+
}
1689+
16491690
}

0 commit comments

Comments
 (0)