Skip to content

Commit 033622d

Browse files
Praveena2607Praveena2607
committed
BigQuery Sink Plugin - Automated test Scenarios
1 parent 6f3af0b commit 033622d

File tree

7 files changed

+645
-3
lines changed

7 files changed

+645
-3
lines changed

‎src/e2e-test/features/bigquery/sink/BigQuerySinkError.feature‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,16 @@ Feature: BigQuery sink - Validate BigQuery sink plugin error scenarios
6262
Then Enter BigQuery sink property table name
6363
Then Enter BigQuery property temporary bucket name "bqInvalidTemporaryBucket"
6464
Then Verify the BigQuery validation error message for invalid property "bucket"
65+
66+
@BQ_SINK_TEST
67+
Scenario:Verify BigQuery Sink properties validation errors for incorrect value of reference name
68+
Given Open Datafusion Project to configure pipeline
69+
When Sink is BigQuery
70+
Then Open BigQuery sink properties
71+
Then Enter BigQuery property reference name "bqInvalidRefName"
72+
Then Enter BigQuery property projectId "projectId"
73+
Then Enter BigQuery property datasetProjectId "projectId"
74+
Then Override Service account details if set in environment variables
75+
Then Enter BigQuery property dataset "dataset"
76+
Then Enter BigQuery sink property table name
77+
Then Verify the BigQuery validation error message for reference name "referenceName"

‎src/e2e-test/features/bigquery/sink/BigQueryToBigQuerySink.feature‎

Lines changed: 446 additions & 0 deletions
Large diffs are not rendered by default.

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryBase.java‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,4 +260,163 @@ public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromS
260260
BeforeActions.scenario.write("Number of records transferred from source table to target table:" + count);
261261
Assert.assertEquals(count, countRecordsTarget);
262262
}
263+
264+
@Then("Enter BigQuery source properties partitionFrom and partitionTo")
265+
public void enterBigQuerySourcePropertiespartitionFromandpartitionTo() throws IOException {
266+
CdfBigQueryPropertiesActions.enterPartitionStartDate(new SimpleDateFormat("dd-MM-yyyy").format(new Date()));
267+
CdfBigQueryPropertiesActions.enterPartitionEndDate(new SimpleDateFormat("dd-MM-yyyy")
268+
.format(DateUtils.addDays(new Date(), 1)));
269+
}
270+
@Then("Validate BigQuery source incorrect property error for Partition Start date {string} value {string}")
271+
public void validateBigQuerySourceIncorrectErrorFor(String property, String value) {
272+
CdfBigQueryPropertiesActions.getSchema();
273+
274+
275+
SeleniumHelper.waitElementIsVisible(CdfBigQueryPropertiesLocators.getSchemaButton, 5L);
276+
String tableFullName = StringUtils.EMPTY;
277+
if (property.equalsIgnoreCase("dataset")) {
278+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":" + PluginPropertyUtils.pluginProp(value)
279+
+ "." + TestSetupHooks.bqSourceTable;
280+
} else if (property.equalsIgnoreCase("table")) {
281+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":"
282+
+ PluginPropertyUtils.pluginProp("dataset")
283+
+ "." + PluginPropertyUtils.pluginProp(value);
284+
} else if (property.equalsIgnoreCase("datasetProject")) {
285+
tableFullName = PluginPropertyUtils.pluginProp(value) + ":" + PluginPropertyUtils.pluginProp("dataset")
286+
+ "." + TestSetupHooks.bqSourceTable;
287+
288+
}else if (property.equalsIgnoreCase("partitionFrom")) {
289+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":"
290+
+ PluginPropertyUtils.pluginProp("dataset")
291+
+ "." + PluginPropertyUtils.pluginProp(value);}
292+
293+
String expectedErrorMessage = PluginPropertyUtils.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_PARTITIONSTARTDATE)
294+
.replaceAll("TABLENAME", tableFullName);
295+
String actualErrorMessage = PluginPropertyUtils.findPropertyErrorElement("partitionFrom").getText();
296+
System.out.println(actualErrorMessage);
297+
Assert.assertEquals("Error message mismatch for Partition Start Date", expectedErrorMessage, actualErrorMessage);
298+
String actualColor = PluginPropertyUtils.getErrorColor(PluginPropertyUtils.findPropertyErrorElement("partitionFrom"));
299+
String expectedColor = ConstantsUtil.ERROR_MSG_COLOR;
300+
Assert.assertEquals(expectedColor, actualColor);
301+
}
302+
303+
@Then("Validate BigQuery source incorrect property error for Partition End date {string} value {string}")
304+
public void validateBigQuerySourceIncorrectPartitionenddateErrorFor(String property, String value) {
305+
CdfBigQueryPropertiesActions.getSchema();
306+
SeleniumHelper.waitElementIsVisible(CdfBigQueryPropertiesLocators.getSchemaButton, 5L);
307+
String tableFullName = StringUtils.EMPTY;
308+
if (property.equalsIgnoreCase("dataset")) {
309+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":" + PluginPropertyUtils.pluginProp(value)
310+
+ "." + TestSetupHooks.bqSourceTable;
311+
} else if (property.equalsIgnoreCase("table")) {
312+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":"
313+
+ PluginPropertyUtils.pluginProp("dataset")
314+
+ "." + PluginPropertyUtils.pluginProp(value);
315+
} else if (property.equalsIgnoreCase("datasetProjectId")) {
316+
tableFullName = PluginPropertyUtils.pluginProp(value) + ":" + PluginPropertyUtils.pluginProp("dataset")
317+
+ "." + TestSetupHooks.bqSourceTable;
318+
}else if (property.equalsIgnoreCase("partitionEndDate")) {
319+
tableFullName = PluginPropertyUtils.pluginProp(value) + ":"
320+
+ PluginPropertyUtils.pluginProp("partitionTo")
321+
+ "." + TestSetupHooks.bqSourceTable;
322+
}
323+
324+
String expectedErrorMessage = PluginPropertyUtils.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_PARTITIONENDDATE)
325+
.replaceAll("TABLENAME", tableFullName);
326+
String actualErrorMessage = PluginPropertyUtils.findPropertyErrorElement("partitionTo").getText();
327+
System.out.println(actualErrorMessage);
328+
Assert.assertEquals("Error message mismatch for Partition End Date", expectedErrorMessage, actualErrorMessage);
329+
String actualColor = PluginPropertyUtils.getErrorColor(PluginPropertyUtils.findPropertyErrorElement("partitionTo"));
330+
String expectedColor = ConstantsUtil.ERROR_MSG_COLOR;
331+
Assert.assertEquals(expectedColor, actualColor);
332+
}
333+
334+
@Then("Enter BigQuery source properties referenceName")
335+
public void EnterBigQuerysourcepropertiesreferenceName() throws IOException {
336+
CdfBigQueryPropertiesActions.enterBigQueryReferenceName("#$%^");
337+
338+
}
339+
340+
@Then("Validate BigQuery source incorrect property error for reference name{string} value {string}")
341+
public void validateBigQuerySourceIncorrectPropertyErrorForreferncename(String property, String value) {
342+
CdfBigQueryPropertiesActions.getSchema();
343+
SeleniumHelper.waitElementIsVisible(CdfBigQueryPropertiesLocators.getSchemaButton, 5L);
344+
String tableFullName = StringUtils.EMPTY;
345+
if (property.equalsIgnoreCase("dataset")) {
346+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":" + PluginPropertyUtils.pluginProp(value)
347+
+ "." + TestSetupHooks.bqSourceTable;
348+
} else if (property.equalsIgnoreCase("table")) {
349+
tableFullName = PluginPropertyUtils.pluginProp("projectId") + ":"
350+
+ PluginPropertyUtils.pluginProp("dataset")
351+
+ "." + PluginPropertyUtils.pluginProp(value);
352+
} else if (property.equalsIgnoreCase("datasetProject")) {
353+
tableFullName = PluginPropertyUtils.pluginProp(value) + ":" + PluginPropertyUtils.pluginProp("dataset")
354+
+ "." + TestSetupHooks.bqSourceTable;
355+
}
356+
else if (property.equalsIgnoreCase("referenceName")) {
357+
tableFullName = PluginPropertyUtils.pluginProp(value) + ":" + PluginPropertyUtils.pluginProp("reference")
358+
+ "." + TestSetupHooks.bqSourceTable;
359+
}
360+
String expectedErrorMessage = PluginPropertyUtils.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_REFERENCENAME)
361+
.replaceAll("TABLENAME", tableFullName);
362+
String actualErrorMessage = PluginPropertyUtils.findPropertyErrorElement("referenceName").getText();
363+
364+
Assert.assertEquals(expectedErrorMessage, actualErrorMessage);
365+
String actualColor = PluginPropertyUtils.getErrorColor(PluginPropertyUtils.findPropertyErrorElement("referenceName"));
366+
String expectedColor = ConstantsUtil.ERROR_MSG_COLOR;
367+
Assert.assertEquals(expectedColor, actualColor);
368+
369+
}
370+
371+
@Then("Enter BigQuery source properties filter")
372+
public void EnterBigQuerysourcepropertiesfilter() throws IOException {
373+
CdfBigQueryPropertiesActions.enterFilter("%%%%");
374+
375+
}
376+
377+
@Then("Enter BigQuery source property output schema {string} as macro argument {string}")
378+
public void enterBigQueryPropertyAsMacroArgumentoutputschema(String pluginProperty, String macroArgument) {
379+
SCHEMA_LOCATORS.schemaActions.click();
380+
SCHEMA_LOCATORS.schemaActionType("macro").click();
381+
WaitHelper.waitForElementToBeHidden(SCHEMA_LOCATORS.schemaActionType("macro"), 5);
382+
try {
383+
enterMacro(CdfPluginPropertyLocator.fromPropertyString(pluginProperty).pluginProperty, macroArgument);
384+
} catch (NullPointerException e) {
385+
Assert.fail("CDF_PLUGIN_PROPERTY_MAPPING for '" + pluginProperty + "' not present in CdfPluginPropertyLocator.");
386+
}
387+
}
388+
389+
@Then("Enter BigQuery property reference name {string}")
390+
public void EnterBigQuerypropertyreferencename(String referenceName) throws IOException {
391+
CdfBigQueryPropertiesActions.enterBigQueryReferenceName(PluginPropertyUtils.pluginProp(referenceName));
392+
}
393+
394+
@Then("Verify the BigQuery validation error message for reference name {string}")
395+
public void VerifytheBigQueryvalidationerrormessageforreferencename (String property) {
396+
CdfStudioActions.clickValidateButton();
397+
String expectedErrorMessage;
398+
if (property.equalsIgnoreCase("gcsChunkSize")) {
399+
expectedErrorMessage = PluginPropertyUtils
400+
.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_CHUNKSIZE);
401+
} else if (property.equalsIgnoreCase("bucket")) {
402+
expectedErrorMessage = PluginPropertyUtils
403+
.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_TEMPORARY_BUCKET);
404+
} else if (property.equalsIgnoreCase("table")) {
405+
expectedErrorMessage = PluginPropertyUtils
406+
.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_TABLE_NAME);
407+
} else if (property.equalsIgnoreCase("referenceName")) {
408+
expectedErrorMessage = PluginPropertyUtils
409+
.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_REFERENCENAME);
410+
} else {
411+
expectedErrorMessage = PluginPropertyUtils.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_REFERENCENAME).
412+
replaceAll("referenceName", property.substring(0, 1).toUpperCase() + property.substring(1));
413+
}
414+
String actualErrorMessage = PluginPropertyUtils.findPropertyErrorElement("referenceName").getText();
415+
System.out.println(actualErrorMessage);
416+
Assert.assertEquals(expectedErrorMessage, actualErrorMessage);
417+
String actualColor = PluginPropertyUtils.getErrorColor(PluginPropertyUtils.findPropertyErrorElement("referenceName"));
418+
String expectedColor = ConstantsUtil.ERROR_MSG_COLOR;
419+
Assert.assertEquals(expectedColor, actualColor);
420+
}
421+
263422
}

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQuerySink.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,11 @@
2424
import io.cdap.plugin.utils.E2EHelper;
2525
import io.cucumber.java.en.Then;
2626
import io.cucumber.java.en.When;
27+
import org.apache.commons.lang3.time.DateUtils;
2728

2829
import java.io.IOException;
30+
import java.text.SimpleDateFormat;
31+
import java.util.Date;
2932
import java.util.UUID;
3033

3134
/**
@@ -54,8 +57,14 @@ public void toggleBigQuerySinkPropertyTruncateTableToTrue() {
5457
CdfBigQueryPropertiesActions.clickTruncatableSwitch();
5558
}
5659

60+
@Then("Toggle BigQuery sink property reuire partition filter to true")
61+
public void toggleBigQuerysinkpropertyreuirepartitionfiltertotrue() {
62+
CdfBigQueryPropertiesActions.toggleRequirePartitionFilter();
63+
}
64+
5765
@Then("Toggle BigQuery sink property updateTableSchema to true")
5866
public void toggleBigQuerySinkPropertyUpdateTableSchemaToTrue() {
67+
5968
CdfBigQueryPropertiesActions.clickUpdateTable();
6069
}
6170

‎src/e2e-test/java/io/cdap/plugin/utils/E2ETestConstants.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,9 @@ public class E2ETestConstants {
1717
public static final String ERROR_MSG_BQ_INCORRECT_CHUNKSIZE = "errorMessageIncorrectBQChunkSize";
1818
public static final String ERROR_MSG_BQ_INCORRECT_TEMPORARY_BUCKET = "errorMessageIncorrectBQBucketName";
1919
public static final String ERROR_MSG_BQ_INCORRECT_PROPERTY = "errorMessageIncorrectBQProperty";
20+
public static final String ERROR_MSG_INCORRECT_PARTITIONSTARTDATE= "errorMessageIncorrectPartitionStartDate";
21+
public static final String ERROR_MSG_INCORRECT_PARTITIONENDDATE= "errorMessageIncorrectPartitionEndDate";
22+
public static final String ERROR_MSG_INCORRECT_REFERENCENAME= "errorMessageInvalidReferenceName";
23+
public static final String ERROR_MSG_INCORRECT_FILTER= "errorMessageIncorrectRegexPathFilter";
24+
public static final String JSON_STRING_VALUE ="jsonStringValue";
2025
}

‎src/e2e-test/resources/errorMessage.properties‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,8 @@ errorMessageMultipleFileWithoutClearDefaultSchema=Found a row with 4 fields when
3333
errorMessageInvalidSourcePath=Invalid bucket name in path 'abc@'. Bucket name should
3434
errorMessageInvalidDestPath=Invalid bucket name in path 'abc@'. Bucket name should
3535
errorMessageInvalidEncryptionKey=CryptoKeyName.parse: formattedString not in valid format: Parameter "abc@" must be
36+
errorMessageIncorrectPartitionStartDate=08-12-2024 is not in a valid format. Enter valid date in format: yyyy-MM-dd
37+
errorMessageIncorrectPartitionEndDate=09-12-2024 is not in a valid format. Enter valid date in format: yyyy-MM-dd
38+
errorMessageInvalidReferenceName=Invalid reference name 'invalidRef&^*&&*'. Supported characters are: letters, numbers, and '_', '-', '.', or '$'.
39+
errorLogsMessageInvalidFilter=Spark Program 'phase-1' failed.
3640

‎src/e2e-test/resources/pluginParameters.properties‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ bqFuturePartitionEndDate=2099-10-02
179179
bqTruncateTableTrue=True
180180
bqUpdateTableSchemaTrue=True
181181
clusterValue=transaction_date
182+
jsonStringValue=transaction_uid.field
183+
jobLabelKey=transaction_uid
184+
jobLabelValue=transaction_uid:redis
182185
TableKey=PersonID
183186
bqSourceTable=dummy
184187
bqCreateTableQueryFile=testdata/BigQuery/BigQueryCreateTableQuery.txt
@@ -193,6 +196,7 @@ bqSourceSchemaDatatype=[{"key":"transaction_info","value":"boolean"},{"key":"tra
193196
{"key":"difference","value":"array"},{"key":"Userdata","value":"record"}]
194197
bqPartitionField=Month_of_Joining
195198
bqPartitionFieldTime=transaction_date
199+
partitionFilter=Customer_Exists
196200
bqRangeStart=1
197201
bqRangeEnd=10
198202
bqRangeInterval=2
@@ -214,7 +218,7 @@ bqUpdateDedupeExpectedFile=testdata/BigQuery/BQUpdateDedupeFile
214218
bqInsertExpectedFile=testdata/BigQuery/BQInsertIntFile
215219
relationTableKeyValue=Name
216220
dedupeByOrder=ASC
217-
dedupeByValue=ID
221+
dedupeByValue=transaction_time
218222
dedupeByValueUpsert=Price
219223
rangeStartValue=2
220224
rangeEndValue=3
@@ -231,7 +235,7 @@ bqDifferentRecordFile=testdata/BigQuery/BQDifferentRecordNameFile
231235
bqDateExpectedFile=testdata/BigQuery/BQDateFile
232236
bqDateTimeExpectedFile=testdata/BigQuery/BQDateTimeFile
233237
bqTimeStampExpectedFile=testdata/BigQuery/BQTimeStampFile
234-
bqPartitionFieldDate=transaction_date
238+
bqPartitionFieldDate=transaction_date,transaction_time
235239
## BIGQUERY-PLUGIN-PROPERTIES-END
236240

237241
## PUBSUBSINK-PLUGIN-PROPERTIES-START
@@ -243,6 +247,7 @@ pubSubErrorThreshold=0
243247
pubSubStringValue=one
244248
pubSubNegativeValue=-100
245249
pubsubDelimiter=@
250+
JasonString=name.first
246251
## PUBSUBSINK-PLUGIN-PROPERTIES-END
247252

248253
## GCSDELETE-PLUGIN-PROPERTIES-START
@@ -331,7 +336,8 @@ bqExecuteDMLUpsert=MERGE `PROJECT_NAME.DATASET.TABLENAME` as T \
331336
UPDATE SET UID = 'UPDATED RECORD' \
332337
WHEN NOT MATCHED THEN \
333338
INSERT ROW
334-
bqExecuteCountDMLUpsertInsert=SELECT COUNT(*) FROM `PROJECT_NAME.DATASET.TABLENAME` WHERE Id=101 AND Value=5000 AND UID='INSERTED RECORD'
339+
bqExecuteCountDMLUpsertInsert=SELECT COUNT(*) FROM `PROJECT_NAME.\
340+
.TABLENAME` WHERE Id=101 AND Value=5000 AND UID='INSERTED RECORD'
335341
bqExecuteCountDMLUpsertUpdate=SELECT COUNT(*) FROM `PROJECT_NAME.DATASET.TABLENAME` WHERE Id=101 AND Value=5000 AND UID='UPDATED RECORD'
336342
bqExecuteInsertFile=testdata/BQExecute/BQExecuteInsertFile
337343
## BQEXECUTE-PLUGIN-PROPERTIES-END

0 commit comments

Comments
 (0)