Merge branch 'develop' into e2e_bigquerymultitable

data-integrations · Dec 5, 2023 · 91af24c · 91af24c
2 parents 568fff4 + 00d2daa
commit 91af24c
Show file tree

Hide file tree

Showing 15 changed files with 941 additions and 11 deletions.
diff --git a/pom.xml b/pom.xml
@@ -1217,16 +1217,6 @@
           <version>1.2.8</version>
           <scope>runtime</scope>
         </dependency>
-        <dependency>
-          <groupId>com.google.apis</groupId>
-          <artifactId>google-api-services-storage</artifactId>
-          <version>v1-rev20220604-1.32.1</version>
-        </dependency>
-        <dependency>
-          <groupId>com.google.cloud</groupId>
-          <artifactId>google-cloud-storage</artifactId>
-          <version>2.8.0</version>
-        </dependency>
       </dependencies>
 
     </profile>

diff --git a/src/e2e-test/features/bigquery/sink/BigQueryToBigQueryAdditional.feature b/src/e2e-test/features/bigquery/sink/BigQueryToBigQueryAdditional.feature
diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java
@@ -27,7 +27,8 @@
   features = {"src/e2e-test/features"},
   glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
     "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
-  tags = {"@BigQuery_Sink"},
+  tags = {"@BigQuery_Sink and not @CDAP-20830"},
+  //TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
   monochrome = true,
   plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink",
     "json:target/cucumber-reports/cucumber-bigquery-sink.json",

diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunnerRequired.java b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunnerRequired.java
@@ -29,6 +29,7 @@
     "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
   tags = {"@BigQuery_Sink_Required"},
   monochrome = true,
+  //TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
   plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink-required",
     "json:target/cucumber-reports/cucumber-bigquery-sink-required.json",
     "junit:target/cucumber-reports/cucumber-bigquery-sink-required.xml"}

diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties
@@ -223,6 +223,31 @@ bqmtInvalidSinkReferenceName=(*^*&*
 bqmtInvalidTemporaryBucket=$#%$
 bqExpectedFile=testdata/BigQuery/BQExistingTableFile
 bgInsertDatatypeFile=testdata/BigQuery/BQInsertDatatypeFile
+TableKeyUpsert=ID
+TableKeyInsert=ID
+bqUpsertExpectedFile=testdata/BigQuery/BQUpsertTableFile
+bqUpdateDedupeExpectedFile=testdata/BigQuery/BQUpdateDedupeFile
+bqInsertExpectedFile=testdata/BigQuery/BQInsertIntFile
+relationTableKeyValue=Name
+dedupeByOrder=ASC
+dedupeByValue=ID
+dedupeByValueUpsert=Price
+rangeStartValue=2
+rangeEndValue=3
+rangeIntervalValue=1
+partitionByFieldValue=ID
+bqPartitionFieldDateTime=transaction_dt
+bqPartitionFieldTimeStamp=updated_on
+bqSourceTable2=dummy
+dedupeBy=DESC
+TableKeyDedupe=Name
+Directive_Drop=testdata/BigQuery/test_diffschema_record-cdap-data-pipeline.json
+bqUpsertDedupeFile=testdata/BigQuery/BQUpsertDedupeFile
+bqDifferentRecordFile=testdata/BigQuery/BQDifferentRecordNameFile
+bqDateExpectedFile=testdata/BigQuery/BQDateFile
+bqDateTimeExpectedFile=testdata/BigQuery/BQDateTimeFile
+bqTimeStampExpectedFile=testdata/BigQuery/BQTimeStampFile
+bqPartitionFieldDate=transaction_date
 ## BIGQUERY-PLUGIN-PROPERTIES-END
 
 ## PUBSUBSINK-PLUGIN-PROPERTIES-START

diff --git a/src/e2e-test/resources/testdata/BigQuery/BQDateFile b/src/e2e-test/resources/testdata/BigQuery/BQDateFile
@@ -0,0 +1,3 @@
+{"Firstname":"singh","ID":"Gurgaon","transaction_date":"2021-02-22","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
+{"Firstname":"Neera","ID":"Agra","transaction_date":"2021-02-20","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
+{"Firstname":"","ID":"Noida","transaction_date":"2021-02-21","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQDateTimeFile b/src/e2e-test/resources/testdata/BigQuery/BQDateTimeFile
@@ -0,0 +1,3 @@
+{"Firstname":"singh","ID":"Gurgaon","transaction_date":"2021-02-22","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
+{"Firstname":"Neera","ID":"Agra","transaction_date":"2021-02-20","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
+{"Firstname":"","ID":"Noida","transaction_date":"2021-02-21","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQDifferentRecordNameFile b/src/e2e-test/resources/testdata/BigQuery/BQDifferentRecordNameFile
@@ -0,0 +1,2 @@
+{"ID":1,"Name":"string_1","Price":0.1}
+{"ID":1,"Name":"string_1","Price":0.1}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQInsertIntFile b/src/e2e-test/resources/testdata/BigQuery/BQInsertIntFile
@@ -0,0 +1,2 @@
+{"Customer_Exists":true,"ID":3,"Name":"Rajan Kumar","Price":100.0}
+{"Customer_Exists":true,"ID":3,"Name":"Rajan Kumar","Price":100.0}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQTimeStampFile b/src/e2e-test/resources/testdata/BigQuery/BQTimeStampFile
@@ -0,0 +1,3 @@
+{"Firstname":"singh","ID":"Gurgaon","transaction_date":"2021-02-22","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
+{"Firstname":"Neera","ID":"Agra","transaction_date":"2021-02-20","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
+{"Firstname":"","ID":"Noida","transaction_date":"2021-02-21","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQUpdateDedupeFile b/src/e2e-test/resources/testdata/BigQuery/BQUpdateDedupeFile
@@ -0,0 +1,2 @@
+{"Customer_Exists":true,"ID":0,"Name":"string_0","Price":0.0}
+{"Customer_Exists":true,"ID":1,"Name":"string_1","Price":0.1}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQUpsertDedupeFile b/src/e2e-test/resources/testdata/BigQuery/BQUpsertDedupeFile
@@ -0,0 +1,3 @@
+{"Customer_Exists":false,"ID":3,"Name":"string_3","Price":0.3}
+{"Customer_Exists":false,"ID":2,"Name":"string_1","Price":0.2}
+{"Customer_Exists":true,"ID":0,"Name":"string_0","Price":0.0}
diff --git a/src/e2e-test/resources/testdata/BigQuery/BQUpsertTableFile b/src/e2e-test/resources/testdata/BigQuery/BQUpsertTableFile
@@ -0,0 +1,2 @@
+{"Customer_Exists":true,"ID":5,"Name":"Raja","Price":500.0}
+{"Customer_Exists":false,"ID":6,"Name":"Tom","Price":100.0}
diff --git a/src/e2e-test/resources/testdata/BigQuery/test_diffschema_record-cdap-data-pipeline.json b/src/e2e-test/resources/testdata/BigQuery/test_diffschema_record-cdap-data-pipeline.json
@@ -0,0 +1,183 @@
+{
+    "name": "test_diffschema_record",
+    "description": "Data Pipeline Application",
+    "artifact": {
+        "name": "cdap-data-pipeline",
+        "version": "6.10.0-SNAPSHOT",
+        "scope": "SYSTEM"
+    },
+    "config": {
+        "resources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "driverResources": {
+            "memoryMB": 2048,
+            "virtualCores": 1
+        },
+        "connections": [
+            {
+                "from": "BigQuery",
+                "to": "Wrangler"
+            },
+            {
+                "from": "Wrangler",
+                "to": "BigQuery3"
+            },
+            {
+                "from": "BigQuery2",
+                "to": "BigQuery3"
+            }
+        ],
+        "comments": [],
+        "postActions": [],
+        "properties": {},
+        "processTimingEnabled": true,
+        "stageLoggingEnabled": false,
+        "stages": [
+            {
+                "name": "BigQuery",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsource",
+                    "label": "BigQuery",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.22.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "project": "cdf-athena",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "referenceName": "bq_ref",
+                        "dataset": "bq_automation",
+                        "table": "bqSourceTableMore",
+                        "enableQueryingViews": "false",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]},{\"name\":\"TableName\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                },
+                "outputSchema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]},{\"name\":\"TableName\",\"type\":[\"string\",\"null\"]}]}",
+                "id": "BigQuery",
+                "type": "batchsource",
+                "label": "BigQuery",
+                "icon": "fa-plug"
+            },
+            {
+                "name": "Wrangler",
+                "plugin": {
+                    "name": "Wrangler",
+                    "type": "transform",
+                    "label": "Wrangler",
+                    "artifact": {
+                        "name": "wrangler-transform",
+                        "version": "4.10.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "field": "*",
+                        "precondition": "false",
+                        "directives": "drop :TableName",
+                        "on-error": "fail-pipeline",
+                        "schema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
+                        "workspaceId": "7038fc39-732e-4d75-8d3f-db6cfe5a11d8"
+                    }
+                },
+                "outputSchema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
+                "inputSchema": [
+                    {
+                        "name": "BigQuery",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]},{\"name\":\"TableName\",\"type\":[\"string\",\"null\"]}]}"
+                    }
+                ],
+                "id": "Wrangler",
+                "type": "transform",
+                "label": "Wrangler",
+                "icon": "icon-DataPreparation"
+            },
+            {
+                "name": "BigQuery3",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsink",
+                    "label": "BigQuery3",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.22.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "project": "auto-detect",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "dataset": "bq_automation",
+                        "table": "New_target_table_combine",
+                        "operation": "insert",
+                        "truncateTable": "false",
+                        "allowSchemaRelaxation": "false",
+                        "location": "US",
+                        "createPartitionedTable": "false",
+                        "partitioningType": "TIME",
+                        "partitionFilterRequired": "false",
+                        "schema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
+                    }
+                },
+                "outputSchema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
+                "inputSchema": [
+                    {
+                        "name": "Wrangler",
+                        "schema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
+                    },
+                    {
+                        "name": "BigQuery2",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
+                    }
+                ],
+                "id": "BigQuery3",
+                "type": "batchsink",
+                "label": "BigQuery3",
+                "icon": "fa-plug"
+            },
+            {
+                "name": "BigQuery2",
+                "plugin": {
+                    "name": "BigQueryTable",
+                    "type": "batchsource",
+                    "label": "BigQuery2",
+                    "artifact": {
+                        "name": "google-cloud",
+                        "version": "0.22.0-SNAPSHOT",
+                        "scope": "SYSTEM"
+                    },
+                    "properties": {
+                        "useConnection": "false",
+                        "project": "cdf-athena",
+                        "serviceAccountType": "filePath",
+                        "serviceFilePath": "auto-detect",
+                        "referenceName": "bq_test",
+                        "dataset": "bq_automation",
+                        "table": "bqSourceTableLess",
+                        "enableQueryingViews": "false",
+                        "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
+                    }
+                },
+                "outputSchema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
+                "id": "BigQuery2",
+                "type": "batchsource",
+                "label": "BigQuery2",
+                "icon": "fa-plug"
+            }
+        ],
+        "schedule": "0 1 */1 * *",
+        "engine": "spark",
+        "numOfRecordsPreview": 100,
+        "rangeRecordsPreview": {
+            "min": 1,
+            "max": "5000"
+        },
+        "maxConcurrentRuns": 1
+    },
+    "version": "fe4ee1e3-6380-11ee-8217-0000003390c8"
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"ID":1,"Name":"string_1","Price":0.1}
		{"ID":1,"Name":"string_1","Price":0.1}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"Customer_Exists":true,"ID":3,"Name":"Rajan Kumar","Price":100.0}
		{"Customer_Exists":true,"ID":3,"Name":"Rajan Kumar","Price":100.0}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"Customer_Exists":true,"ID":0,"Name":"string_0","Price":0.0}
		{"Customer_Exists":true,"ID":1,"Name":"string_1","Price":0.1}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"Customer_Exists":true,"ID":5,"Name":"Raja","Price":500.0}
		{"Customer_Exists":false,"ID":6,"Name":"Tom","Price":100.0}