bq validation

cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTime.feature

@@ -57,7 +57,7 @@ Feature: CloudMySql Sink - Run time scenarios
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target CloudSQLMySQL table is equal to the values from source BigQuery table
+    Then Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source BigQuery table
 
   @BQ_SOURCE_TEST @CLOUDMYSQL_TEST_TABLE @PLUGIN-1526
   Scenario: To verify data is getting transferred from BigQuery source to CloudMySql sink successfully when connection arguments are set
@@ -102,7 +102,8 @@ Feature: CloudMySql Sink - Run time scenarios
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target CloudSQLMySQL table is equal to the values from source BigQuery table
+    Then Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source BigQuery table
+
 
   @BQ_SOURCE_TEST @CLOUDMYSQL_TEST_TABLE @PLUGIN-1526
   Scenario: To verify data is getting transferred from BigQuery source to CloudSMySQL sink with Advanced property Connection timeout
@@ -147,6 +148,7 @@ Feature: CloudMySql Sink - Run time scenarios
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target CloudSQLMySQL table is equal to the values from source BigQuery table
+    Then Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source BigQuery table
+
 
 

cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature

@@ -63,7 +63,7 @@ Feature: CloudMySql Sink - Run time scenarios (macro)
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target CloudSQLMySQL table is equal to the values from source BigQuery table
+    Then Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source BigQuery table
 
   @BQ_SOURCE_TEST @CLOUDMYSQL_TEST_TABLE @PLUGIN-1526
   Scenario: To verify data is getting transferred from BigQuery source to CloudMySql sink using macro arguments in basic section
@@ -108,7 +108,7 @@ Feature: CloudMySql Sink - Run time scenarios (macro)
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target CloudSQLMySQL table is equal to the values from source BigQuery table
+    Then Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source BigQuery table
 
   @BQ_SOURCE_TEST @CLOUDMYSQL_TEST_TABLE
   Scenario: Verify pipeline failure message in logs when user provides invalid Table Name of plugin with Macros
@@ -232,4 +232,4 @@ Feature: CloudMySql Sink - Run time scenarios (macro)
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target CloudSQLMySQL table is equal to the values from source BigQuery table
+    Then Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source BigQuery table

cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature

@@ -56,8 +56,7 @@ Feature: CloudMySql - Verify data transfer from CloudMySql source to BigQuery si
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target BigQuery table is equal to the values from source table
-
+    Then Validate the values of records transferred to target Big Query table is equal to the values from source table
   @CLOUDMYSQL_SOURCE_DATATYPES_TEST @BQ_SINK_TEST
   Scenario: To verify data is getting transferred from CloudMySql source to BigQuery sink successfully with all datatypes
     Given Open Datafusion Project to configure pipeline
@@ -99,7 +98,8 @@ Feature: CloudMySql - Verify data transfer from CloudMySql source to BigQuery si
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target BigQuery table is equal to the values from source table
+    Then Validate the values of records transferred to target Big Query table is equal to the values from source table
+
 
   @CLOUDMYSQL_SOURCE_DATATYPES_TEST @CLOUDMYSQL_SINK_TEST @PLUGIN-20670
   Scenario: To verify data is getting transferred from CloudMySql source to CloudMySql sink successfully

cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTimeMacro.feature

@@ -212,7 +212,7 @@ Feature: CloudMySql - Verify CloudMySql plugin data transfer with macro argument
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target BigQuery table is equal to the values from source table
+    Then Validate the values of records transferred to target Big Query table is equal to the values from source table
 
   @CLOUDMYSQL_SOURCE_TEST @BQ_SINK_TEST
   Scenario: To verify data is getting transferred from CloudMySql source to BigQuery sink successfully using macro arguments in basic section
@@ -256,7 +256,8 @@ Feature: CloudMySql - Verify CloudMySql plugin data transfer with macro argument
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target BigQuery table is equal to the values from source table
+    Then Validate the values of records transferred to target Big Query table is equal to the values from source table
+
 
   @CLOUDMYSQL_SOURCE_TEST @BQ_SINK_TEST
   Scenario: To verify data is getting transferred from CloudMySql source to BigQuery sink successfully using macro arguments in advance section
@@ -304,4 +305,5 @@ Feature: CloudMySql - Verify CloudMySql plugin data transfer with macro argument
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
-    Then Validate the values of records transferred to target BigQuery table is equal to the values from source table
+    Then Validate the values of records transferred to target Big Query table is equal to the values from source table
+

cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySql/BQValidation.java

@@ -35,6 +35,7 @@ public class BQValidation {
      * @return true if the values in source and target side are equal
      */
 
+
     public static boolean validateBQAndDBRecordValues(String sourceTable, String targetTable)
             throws SQLException, ClassNotFoundException, IOException, InterruptedException, ParseException {
         getBigQueryTableData(sourceTable, bigQueryRows);
@@ -198,12 +199,12 @@ public static boolean compareResultSetWithJsonData(ResultSet rsSource, List<Json
                         break;
 
                     case Types.TIMESTAMP:
-//                        String sourceTS= String.valueOf(rsSource.getTimestamp(currentColumnCount));
-//                        String targetTS=bigQueryData.get(jsonObjectIdx).get(columnName).getAsString();
-//                        LocalDateTime timestamp = LocalDateTime.parse(targetTS, DateTimeFormatter.ISO_DATE_TIME);
-//                        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S");
-//                        String formattedTimestamp = timestamp.format(formatter);
-//                        Assert.assertEquals(sourceTS, formattedTimestamp);
+                        String sourceTS= String.valueOf(rsSource.getTimestamp(currentColumnCount));
+                        String targetTS=bigQueryData.get(jsonObjectIdx).get(columnName).getAsString();
+                        LocalDateTime timestamp = LocalDateTime.parse(targetTS, DateTimeFormatter.ISO_DATE_TIME);
+                        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S");
+                        String formattedTimestamp = timestamp.format(formatter);
+                        Assert.assertEquals(sourceTS, formattedTimestamp);
                         break;
 
                     default:

cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySql/stepsdesign/CloudMysql.java

@@ -1,20 +1,23 @@
 package io.cdap.plugin.CloudMySql.stepsdesign;
 
+import io.cdap.e2e.pages.actions.CdfPipelineRunAction;
+import io.cdap.e2e.utils.BigQueryClient;
 import io.cdap.e2e.utils.CdfHelper;
 import io.cdap.e2e.utils.PluginPropertyUtils;
 import io.cdap.plugin.CloudMySqlClient;
 import io.cucumber.java.en.Then;
 import org.junit.Assert;
 import stepsdesign.BeforeActions;
-import io.cdap.e2e.utils.PluginPropertyUtils;
+import io.cdap.plugin.CloudMySql.BQValidation;
 
+import java.io.IOException;
 import java.sql.SQLException;
+import java.text.ParseException;
 
 /**
  *  CloudSqlMySql Plugin related step design.
  */
 public class CloudMysql implements CdfHelper {
-
     @Then("Validate the values of records transferred to target table is equal to the values from source table")
     public void validateTheValuesOfRecordsTransferredToTargetTableIsEqualToTheValuesFromSourceTable()
             throws SQLException, ClassNotFoundException {
@@ -27,5 +30,37 @@ public void validateTheValuesOfRecordsTransferredToTargetTableIsEqualToTheValues
         Assert.assertTrue("Value of records transferred to the target table should be equal to the value " +
                 "of the records in the source table", recordsMatched);
     }
+
+    @Then("Validate the values of records transferred to target Big Query table is equal to the values from source table")
+    public void validateTheValuesOfRecordsTransferredToTargetBigQueryTableIsEqualToTheValuesFromSourceTable()
+            throws IOException, InterruptedException, IOException, SQLException, ClassNotFoundException, ParseException {
+        int targetBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqTargetTable"));
+        BeforeActions.scenario.write("No of Records Transferred to BigQuery:" + targetBQRecordsCount);
+        Assert.assertEquals("Out records should match with target BigQuery table records count",
+                CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), targetBQRecordsCount);
+
+        boolean recordsMatched = BQValidation.validateDBAndBQRecordValues(
+                PluginPropertyUtils.pluginProp("sourceTable"),
+                PluginPropertyUtils.pluginProp("bqTargetTable"));
+        Assert.assertTrue("Value of records transferred to the target table should be equal to the value " +
+                "of the records in the source table", recordsMatched);
+    }
+
+    @Then("Validate the values of records transferred to target CloudSQLMySql table is equal to the values from source " +
+            "BigQuery table")
+    public void validateTheValuesOfRecordsTransferredToTargetCloudSQLMySqlTableIsEqualToTheValuesFromSourceBigQueryTable()
+            throws IOException, InterruptedException, IOException, SQLException, ClassNotFoundException, ParseException, ParseException {
+        int sourceBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqSourceTable"));
+        BeforeActions.scenario.write("No of Records from source BigQuery table:" + sourceBQRecordsCount);
+        Assert.assertEquals("Out records should match with target PostgreSQL table records count",
+                CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), sourceBQRecordsCount);
+
+        boolean recordsMatched = BQValidation.validateBQAndDBRecordValues(
+                PluginPropertyUtils.pluginProp("bqSourceTable"),
+                PluginPropertyUtils.pluginProp("targetTable"));
+        Assert.assertTrue("Value of records transferred to the target table should be equal to the value " +
+                "of the records in the source table", recordsMatched);
+    }
+
 }
 

cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySqlClient.java

@@ -21,10 +21,9 @@ public static Connection getCloudSqlConnection() throws ClassNotFoundException,
         String password = System.getenv("CLOUDSQLMYSQL_PASSWORD");
 
         String jdbcUrl = String.format(
-                "jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=%s&password=%s",
+                PluginPropertyUtils.pluginProp("jdbcURL"),
                 database, instanceConnectionName, username, password);
         Connection conn = DriverManager.getConnection(jdbcUrl);
-        System.out.println("connected to database");
         return conn;
     }
 

cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -24,16 +24,6 @@
  */
 
 public class TestSetupHooks {
-    public static void main(String[] args) throws SQLException, ClassNotFoundException, IOException, InterruptedException {
-//       setTableName();
-//       createCloudMysqlTestTable();
-//        createTables();
-//        createDatatypesTable();
-        createTempSourceBQTable();
-        //createCloudMysqlTestTable();
-//        createTempSourceBQTable();
-//        createCloudMysqlTestTable();
-    }
     public static void setTableName() {
         String randomString = RandomStringUtils.randomAlphabetic(10);
         String sourceTableName = String.format("SourceTable_%s", randomString);
@@ -60,7 +50,7 @@ public static void initializeDBProperties() {
 
     @Before(order = 2, value = "@CLOUDMYSQL_SOURCE_TEST")
     public static void createTables() throws SQLException, ClassNotFoundException {
-        CloudMySqlClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"));
+        CloudMySqlClient.createSourceTable(PluginPropertyUtils.pluginProp("CLOUDMYSQL_SOURCE_TEST"));
         CloudMySqlClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"));
     }
 

cloudsql-mysql-plugin/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt

@@ -1,3 +1,3 @@
 insert into `DATASET.TABLE_NAME` (COL1, COL2, COL3, COL4, COL6, COL8, COL9, COL10) values
-(b'01011011','priya','2021-01-28',1.110,'2019-03-10 04:50:01 UTC',false,92233720,'21:26:00'),
-(b'01011011','surya','2021-01-21',1.110000001,'2018-03-10 04:50:01 UTC',true,92233729,'20:26:00');
+(b'01011011','priya','2021-01-27',1.110,'2019-03-10 04:50:01 UTC',false,92233720,'21:26:00'),
+(b'01011011','surya','2021-01-27',1.110000001,'2018-03-10 04:50:01 UTC',true,92233729,'20:26:00');

cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties

@@ -15,10 +15,11 @@ CloudMySqlImportQuery=select * from mytable
 fetchSize=1000
 NumSplits=1
 SplitBy=ID
+jdbcURL=jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=%s&password=%s
 projectId=cdf-athena
 datasetprojectId=cdf-athena
 BQReferenceName=reference
-targetTable=mytable4
+targetTable=mytable5
 bqDatasetId=1234
 dataset=test_automation
 bqSourceTable=mysql
@@ -29,6 +30,7 @@ pass=PASS
 invalidUserName=testUser
 invalidPassword=testPassword
 invalidTable=data
+jdbcURL=jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=%s&password=%s
 CloudMySqlDriverName=cloudsql-mysql
 bqTruncateTable=truncateTable
 bqUpdateTableSchema=updateSchema