Skip to content

Commit d6c1956

Browse files
psainicsAnkitCLI
psainics
authored and
AnkitCLI
committed
bqmt fixes
1 parent 601f62b commit d6c1956

File tree

4 files changed

+72
-33
lines changed

4 files changed

+72
-33
lines changed

‎src/e2e-test/features/bigquery/multitablesink/BigQueryMultiTableAdditional.feature‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
@BigQueryMultiTable_Sink
1616
Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable successful data transfer
1717

18-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_DELETE_TABLES_TEST
18+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_TWO_SINK_BQMT_TEST
1919
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two new tables
2020
Given Open Datafusion Project to configure pipeline
2121
When Expand Plugin group in the LHS plugins list: "Source"
@@ -89,7 +89,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
8989
Then Verify the pipeline status is "Succeeded"
9090
Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
9191

92-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
92+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
9393
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables
9494
Given Open Datafusion Project to configure pipeline
9595
When Expand Plugin group in the LHS plugins list: "Source"
@@ -131,7 +131,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
131131
Then Verify the pipeline status is "Succeeded"
132132
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
133133

134-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
134+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
135135
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables using truncate
136136
Given Open Datafusion Project to configure pipeline
137137
When Expand Plugin group in the LHS plugins list: "Source"
@@ -174,7 +174,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
174174
Then Verify the pipeline status is "Succeeded"
175175
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
176176

177-
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
177+
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
178178
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables after updating schema
179179
Given Open Datafusion Project to configure pipeline
180180
When Expand Plugin group in the LHS plugins list: "Source"

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package io.cdap.plugin.bigquery.stepsdesign;
1818

1919
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
2021
import io.cucumber.java.en.Then;
2122
import org.junit.Assert;
2223

@@ -36,7 +37,7 @@ public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
3637
PluginPropertyUtils.pluginProp("bqSourceTable2"));
3738

3839
List<String> targetTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqTargetTable"),
39-
PluginPropertyUtils.pluginProp("bqTargetTable2"));
40+
PluginPropertyUtils.pluginProp("bqTargetTable2"));
4041
boolean recordsMatched = BigQueryMultiTableValidation.validateBQToBigQueryMultiTable(sourceTables, targetTables);
4142
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
4243
"of the records in the source table", recordsMatched);
@@ -46,8 +47,9 @@ public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
4647
public void validateDataTransferredFromBigQueryToBigQueryMultiTableInOneTableIsEqual()
4748
throws IOException, InterruptedException {
4849
boolean recordsMatched = BigQueryMultiTableValidation.
49-
validateBQToBigQueryMultiTable(Collections.singletonList(PluginPropertyUtils.pluginProp("bqSourceTable")),
50-
Collections.singletonList(PluginPropertyUtils.pluginProp("bqTargetTable")));
50+
validateBQToBigQueryMultiTable(Collections.singletonList(TestSetupHooks.bqSourceTable),
51+
Collections.singletonList(PluginPropertyUtils.pluginProp
52+
("bqTargetTable")));
5153
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
5254
"of the records in the source table", recordsMatched);
5355
}

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 62 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -294,16 +294,16 @@ public static void createTempSourceBQTable() throws IOException, InterruptedExce
294294
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
295295
}
296296

297-
@After(order = 1, value = "@BQ_SOURCE_TEST or @BQ_PARTITIONED_SOURCE_TEST or @BQ_SOURCE_DATATYPE_TEST or " +
297+
@After(order = 2, value = "@BQ_SOURCE_TEST or @BQ_PARTITIONED_SOURCE_TEST or @BQ_SOURCE_DATATYPE_TEST or " +
298298
"@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST or @BQ_EXISTING_SOURCE_TEST or @BQ_EXISTING_SINK_TEST or " +
299299
"@BQ_EXISTING_SOURCE_DATATYPE_TEST or @BQ_EXISTING_SINK_DATATYPE_TEST or @BQ_UPSERT_SOURCE_TEST or " +
300300
"@BQ_NULL_MODE_SOURCE_TEST or @BQ_UPDATE_SOURCE_DEDUPE_TEST or @BQ_INSERT_INT_SOURCE_TEST or " +
301-
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST")
301+
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST or " +
302+
"@BQ_SINGLE_SOURCE_BQMT_TEST")
302303
public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
303304
BigQueryClient.dropBqQuery(bqSourceTable);
304305
PluginPropertyUtils.removePluginProp("bqSourceTable");
305306
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
306-
bqSourceTable = StringUtils.EMPTY;
307307
}
308308

309309
/**
@@ -1554,7 +1554,7 @@ public static void emptyExistingBigTableInstanceAndTableName() {
15541554
bigtableExistingTargetTable = StringUtils.EMPTY;
15551555
}
15561556

1557-
@Before(order = 1, value = "@BQ_EXISTING_TARGET_TEST")
1557+
@Before(order = 2, value = "@BQ_EXISTING_TARGET_TEST")
15581558
public static void createSinkTables() throws IOException, InterruptedException {
15591559
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
15601560
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
@@ -1604,35 +1604,31 @@ public static void createSourceTables() throws IOException, InterruptedException
16041604
PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2);
16051605
}
16061606

1607-
@After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1608-
public static void deleteAllBqTables() throws IOException, InterruptedException {
1609-
BigQueryClient.dropBqQuery(bqSourceTable);
1610-
BigQueryClient.dropBqQuery(bqSourceTable2);
1611-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1612-
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1613-
BigQueryClient.dropBqQuery(bqTargetTable);
1614-
BigQueryClient.dropBqQuery(bqTargetTable2);
1615-
PluginPropertyUtils.removePluginProp("bqSourceTable");
1616-
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1617-
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
1618-
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1619-
BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1620-
BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
1621-
}
1622-
1623-
@After(order = 1, value = "@BQ_SINK_BQMT_TEST")
1607+
// @After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1608+
// public static void deleteAllBqTables() throws IOException, InterruptedException {
1609+
// BigQueryClient.dropBqQuery(bqSourceTable);
1610+
// BigQueryClient.dropBqQuery(bqSourceTable2);
1611+
// bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1612+
// bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1613+
// BigQueryClient.dropBqQuery(bqTargetTable);
1614+
// BigQueryClient.dropBqQuery(bqTargetTable2);
1615+
// PluginPropertyUtils.removePluginProp("bqSourceTable");
1616+
// PluginPropertyUtils.removePluginProp("bqSourceTable2");
1617+
// BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
1618+
// BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1619+
// BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1620+
// BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
1621+
// }
1622+
1623+
@After(order = 2, value = "@BQ_SINK_BQMT_TEST")
16241624
public static void deleteTargetBqmtTable() throws IOException, InterruptedException {
16251625
try {
16261626
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
16271627
BigQueryClient.dropBqQuery(bqTargetTable);
1628-
BigQueryClient.dropBqQuery(bqSourceTable);
16291628
BeforeActions.scenario.write("BQ Target table - " + bqTargetTable + " deleted successfully");
1630-
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " deleted successfully");
1631-
bqTargetTable = StringUtils.EMPTY;
16321629
} catch (BigQueryException e) {
16331630
if (e.getMessage().contains("Not found: Table")) {
16341631
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " does not exist");
1635-
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " does not exist");
16361632
} else {
16371633
Assert.fail(e.getMessage());
16381634
}
@@ -1646,4 +1642,45 @@ public static void deleteTempSource2BQTable() throws IOException, InterruptedExc
16461642
PluginPropertyUtils.removePluginProp("bqSourceTable2");
16471643
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
16481644
}
1645+
1646+
@After(order = 2, value = "@BQ_TWO_SOURCE_BQMT_TEST or @BQ_SOURCE_UPDATE_TEST")
1647+
public static void deleteTwoSourceTables() throws IOException, InterruptedException {
1648+
try {
1649+
BigQueryClient.dropBqQuery(bqSourceTable);
1650+
BigQueryClient.dropBqQuery(bqSourceTable2);
1651+
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " deleted successfully");
1652+
BeforeActions.scenario.write("BQ Source table2 - " + bqSourceTable2 + " deleted successfully");
1653+
PluginPropertyUtils.removePluginProp("bqSourceTable");
1654+
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1655+
1656+
} catch (BigQueryException e) {
1657+
if (e.getMessage().contains("Not found: Table")) {
1658+
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " does not exist");
1659+
BeforeActions.scenario.write("BQ Source Table2 " + bqSourceTable2 + " does not exist");
1660+
} else {
1661+
Assert.fail(e.getMessage());
1662+
}
1663+
}
1664+
}
1665+
1666+
@After(order = 2, value = "@BQ_TWO_SINK_BQMT_TEST")
1667+
public static void deleteTwoTargetTables() throws IOException, InterruptedException {
1668+
try {
1669+
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1670+
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1671+
BigQueryClient.dropBqQuery(bqTargetTable);
1672+
BigQueryClient.dropBqQuery(bqTargetTable2);
1673+
BeforeActions.scenario.write("BQ Target table - " + bqTargetTable + " deleted successfully");
1674+
BeforeActions.scenario.write("BQ Target table2 - " + bqTargetTable2 + " deleted successfully");
1675+
1676+
} catch (BigQueryException e) {
1677+
if (e.getMessage().contains("Not found: Table")) {
1678+
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " does not exist");
1679+
BeforeActions.scenario.write("BQ Target Table2 " + bqTargetTable2 + " does not exist");
1680+
} else {
1681+
Assert.fail(e.getMessage());
1682+
}
1683+
}
1684+
}
1685+
16491686
}

‎src/e2e-test/resources/pluginParameters.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
projectId=cdf-athena
22
datasetprojectId=cdf-athena
3-
dataset=testing_bqmt
3+
dataset=testbq_bqmt
44
wrongSourcePath=gs://00000000-e2e-0014a44f-81be-4501-8360-0ddca192492
55
serviceAccountType=filePath
66
serviceAccount=auto-detect

0 commit comments

Comments
 (0)