From 57e81439f5a4a9977fe2ac80739ff65c6c48250c Mon Sep 17 00:00:00 2001
From: neerajsinghal <Neeraj.singhal@cloudsufi.com>
Date: Thu, 9 Nov 2023 21:50:03 +0530
Subject: [PATCH] e2e test added for gcs multipart upload

---
 pom.xml                                       | 10 +++++
 .../bigquery/source/BigQueryToGCS.feature     | 40 +++++++++++++++++++
 .../common/stepsdesign/TestSetupHooks.java    | 12 +++++-
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 22136d13b8..b264895d60 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1210,6 +1210,16 @@
           <version>1.2.8</version>
           <scope>runtime</scope>
         </dependency>
+        <dependency>
+          <groupId>com.google.apis</groupId>
+          <artifactId>google-api-services-storage</artifactId>
+          <version>v1-rev20220604-1.32.1</version>
+        </dependency>
+        <dependency>
+          <groupId>com.google.cloud</groupId>
+          <artifactId>google-cloud-storage</artifactId>
+          <version>2.8.0</version>
+        </dependency>
       </dependencies>
 
     </profile>
diff --git a/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature b/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature
index e65f6511d4..5de9a43363 100644
--- a/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature
+++ b/src/e2e-test/features/bigquery/source/BigQueryToGCS.feature
@@ -116,3 +116,43 @@ Feature: BigQuery source - Verification of BigQuery to GCS successful data trans
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Verify data is transferred to target GCS bucket
+
+@BQ_SOURCE_DATATYPE_TEST @GCS_SINK_MULTI_PART_UPLOAD
+Scenario:Validate successful records transfer from BigQuery to GCS with bucket having delete multi part upload policy enabled
+  Given Open Datafusion Project to configure pipeline
+  When Expand Plugin group in the LHS plugins list: "Source"
+  When Select plugin: "BigQuery" from the plugins list as: "Source"
+  When Expand Plugin group in the LHS plugins list: "Sink"
+  When Select plugin: "GCS" from the plugins list as: "Sink"
+  Then Navigate to the properties page of plugin: "BigQuery"
+  And Enter input plugin property: "referenceName" with value: "Reference"
+  And Replace input plugin property: "project" with value: "projectId"
+  And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+  And Replace input plugin property: "dataset" with value: "dataset"
+  Then Override Service account details if set in environment variables
+  And Enter input plugin property: "table" with value: "bqSourceTable"
+  Then Click on the Get Schema button
+  Then Validate output schema with expectedSchema "bqSourceSchemaDatatype"
+  Then Validate "BigQuery" plugin properties
+  Then Close the BigQuery properties
+  Then Navigate to the properties page of plugin: "GCS"
+  Then Enter input plugin property: "referenceName" with value: "sourceRef"
+  Then Replace input plugin property: "project" with value: "projectId"
+  Then Enter GCS sink property path
+  Then Select dropdown plugin property: "select-format" with option value: "json"
+  Then Validate "GCS" plugin properties
+  Then Close the Plugin Properties page
+  Then Connect source as "BigQuery" and sink as "GCS" to establish connection
+  Then Save the pipeline
+  Then Preview and run the pipeline
+  Then Wait till pipeline preview is in running state
+  Then Open and capture pipeline preview logs
+  Then Verify the preview run status of pipeline in the logs is "succeeded"
+  Then Close the pipeline logs
+  Then Close the preview
+  Then Deploy the pipeline
+  Then Run the Pipeline in Runtime
+  Then Wait till pipeline is in running state
+  Then Open and capture logs
+  Then Verify the pipeline status is "Succeeded"
+  Then Validate the values of records transferred to GCS bucket is equal to the values from source BigQuery table
diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
index 8c5b1e385f..7e18831e0c 100644
--- a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
+++ b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
@@ -211,7 +211,7 @@ public static void createTargetGCSBucketWithCSVFile() throws IOException, URISyn
     BeforeActions.scenario.write("GCS target bucket name - " + gcsTargetBucketName);
   }
 
-  @After(order = 1, value = "@GCS_SINK_TEST or @GCS_SINK_EXISTING_BUCKET_TEST")
+  @After(order = 1, value = "@GCS_SINK_TEST or @GCS_SINK_EXISTING_BUCKET_TEST or @GCS_SINK_MULTI_PART_UPLOAD")
   public static void deleteTargetBucketWithFile() {
     deleteGCSBucket(gcsTargetBucketName);
     PluginPropertyUtils.removePluginProp("gcsTargetBucketName");
@@ -1030,4 +1030,14 @@ public static void createSinkBQExistingDatatypeTable() throws IOException, Inter
     PluginPropertyUtils.addPluginProp(" bqTargetTable",  bqTargetTable);
     BeforeActions.scenario.write("BQ Target Table " +  bqTargetTable + " updated successfully");
   }
+  private static String createGCSBucketLifeCycle() throws IOException, URISyntaxException {
+    String bucketName = StorageClient.createBucketwithLifeCycle("00000000-e2e-" + UUID.randomUUID(), 30).getName();
+    PluginPropertyUtils.addPluginProp("gcsTargetBucketName", bucketName);
+    return bucketName;
+  }
+
+  @Before(order = 1, value = "@GCS_SINK_MULTI_PART_UPLOAD")
+  public static void createBucketWithLifeCycle() throws IOException, URISyntaxException {
+    gcsTargetBucketName = createGCSBucketLifeCycle();
+    BeforeActions.scenario.write("GCS target bucket name - " + gcsTargetBucketName); }
 }