new scenarios e2e bq

Author: priyabhatnagar25

src/e2e-test/features/bigquery/sink/BigQueryToBigQueryAdditional.feature

@@ -27,7 +27,8 @@
   features = {"src/e2e-test/features"},
   glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
     "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
-  tags = {"@BigQuery_Sink"},
+  tags = {"@BigQuery_Sink and not @CDAP-20830"},
+  //TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
   monochrome = true,
   plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink",
     "json:target/cucumber-reports/cucumber-bigquery-sink.json",

src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java

@@ -0,0 +1,116 @@
+package io.cdap.plugin.bigquery.stepsdesign;
+
+import com.esotericsoftware.minlog.Log;
+import com.google.cloud.bigquery.FieldValueList;
+import com.google.cloud.bigquery.TableResult;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import io.cdap.e2e.utils.BigQueryClient;
+import io.cdap.e2e.utils.PluginPropertyUtils;
+import io.cucumber.core.logging.Logger;
+import io.cucumber.core.logging.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * BigQuery Plugin Existing Table validation.
+ */
+public class BQValidationExistingTables {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BQValidationExistingTables.class);
+  private static final Gson gson = new Gson();
+
+  /**
+   * Validates the actual data in a BigQuery table against the expected data in a JSON file.
+   * @param table    The name of the BigQuery table to retrieve data from.
+   * @param fileName The name of the JSON file containing the expected data.
+   * @return True if the actual data matches the expected data, false otherwise.
+   */
+  public static boolean validateActualDataToExpectedData(String table, String fileName) throws IOException,
+    InterruptedException, URISyntaxException {
+    Map<String, JsonObject> bigQueryMap = new HashMap<>();
+    Map<String, JsonObject> fileMap = new HashMap<>();
+    Path bqExpectedFilePath = Paths.get(BQValidationExistingTables.class.getResource("/" + fileName).toURI());
+
+    getBigQueryTableData(table, bigQueryMap);
+    getFileData(bqExpectedFilePath.toString(), fileMap);
+    boolean isMatched = bigQueryMap.equals(fileMap);
+    return isMatched;
+  }
+
+  /**
+   * Reads a JSON file line by line and populates a map with JSON objects using a specified ID key.
+   *@param fileName The path to the JSON file to be read.
+   * @param fileMap  A map where the extracted JSON objects will be stored with their ID values as keys.
+   */
+
+  public static void getFileData(String fileName, Map<String, JsonObject> fileMap) {
+    try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
+      String line;
+      while ((line = br.readLine()) != null) {
+        JsonObject json = gson.fromJson(line, JsonObject.class);
+        String idKey = getIdKey(json);
+        if (idKey != null) {
+          JsonElement idElement = json.get(idKey);
+          if (idElement.isJsonPrimitive()) {
+            String idValue = idElement.getAsString();
+            fileMap.put(idValue, json);
+          }
+        } else {
+          Log.error("ID key not found");
+        }
+      }
+    } catch (IOException e) {
+      Log.error("Error reading the file: " + e.getMessage());
+    }
+  }
+
+  private static void getBigQueryTableData(String targetTable, Map<String, JsonObject> bigQueryMap)
+    throws IOException, InterruptedException {
+    String dataset = PluginPropertyUtils.pluginProp("dataset");
+    String projectId = PluginPropertyUtils.pluginProp("projectId");
+    String selectQuery = "SELECT TO_JSON(t) FROM `" + projectId + "." + dataset + "." + targetTable + "` AS t";
+    TableResult result = BigQueryClient.getQueryResult(selectQuery);
+
+    for (FieldValueList row : result.iterateAll()) {
+      JsonObject json = gson.fromJson(row.get(0).getStringValue(), JsonObject.class);
+      String idKey = getIdKey(json); // Get the actual ID key from the JSON object
+      if (idKey != null) {
+        JsonElement idElement = json.get(idKey);
+        if (idElement.isJsonPrimitive()) {
+          String id = idElement.getAsString();
+          bigQueryMap.put(id, json);
+        } else {
+          Log.error("Data Mismatched");
+        }
+      }
+    }
+  }
+
+  /**
+   * Retrieves the key for the ID element in the provided JSON object.
+   *
+   * @param json The JSON object to search for the ID key.
+   */
+  private static String getIdKey(JsonObject json) {
+    if (json.has("ID")) {
+      return "ID";
+    } else if (json.has("Name")) {
+      return "Name";
+    } else if (json.has("Price")) {
+      return "Price";
+    } else if (json.has("Customer_Exists")) {
+      return "Customer_Exists";
+    } else {
+      return null;
+    }
+  }
+}

src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BQValidationExistingTables.java

@@ -25,6 +25,7 @@
 import stepsdesign.BeforeActions;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 
 /**
  * BigQuery Plugin validation common step design.
@@ -44,4 +45,13 @@ public void validateTheValuesOfRecordsTransferredToBQsinkIsEqualToTheValuesFromS
     Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
                         "of the records in the source table", recordsMatched);
   }
+
+  @Then("Validate the data transferred from BigQuery to BigQuery with actual And expected file for: {string}")
+  public void validateTheDataFromBQToBQWithActualAndExpectedFileFor(String expectedFile) throws IOException,
+    InterruptedException, URISyntaxException {
+    boolean recordsMatched = BQValidationExistingTables.validateActualDataToExpectedData(
+      PluginPropertyUtils.pluginProp("bqTargetTable"),
+      PluginPropertyUtils.pluginProp(expectedFile));
+    Assert.assertTrue("Value of records in actual and expected file is equal", recordsMatched);
+  }
 }