Merge pull request cdapio#1900 from cloudsufi/xmlReader-cloudbuildfix

Xmlreader e2e cloudbuildfix
cloudsufi · Dec 9, 2024 · da1c3a8 · da1c3a8
2 parents 7793ef2 + ef86151
commit da1c3a8
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 6 deletions.
diff --git a/core-plugins/src/e2e-test/features/xmlReader/XmlReader.feature b/core-plugins/src/e2e-test/features/xmlReader/XmlReader.feature
@@ -44,7 +44,7 @@ Feature:File Sink - Verification of XmlReader plugin to File successful data tra
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
     Then Validate OUT record count is equal to IN record count
-    Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest"
+    Then Validate output file generated by xmlReader and file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest"
 
   @XmlReader_Source_Required @XMLREADER_DELETE_TEST @FILE_SINK_TEST
   Scenario: To verify data is getting transferred from XmlReader to File sink using pattern and delete action
@@ -78,7 +78,7 @@ Feature:File Sink - Verification of XmlReader plugin to File successful data tra
     Then Close the pipeline logs
     Then Validate OUT record count is equal to IN record count
     Then Validate that file gets successfully deleted from the gcs bucket
-    Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest"
+    Then Validate output file generated by xmlReader and file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest"
 
   @XMLREADER_TEST @FILE_SINK_TEST
   Scenario: To verify data is getting transferred from XmlReader to File sink using move action
@@ -112,4 +112,4 @@ Feature:File Sink - Verification of XmlReader plugin to File successful data tra
     Then Close the pipeline logs
     Then Validate OUT record count is equal to IN record count
     Then Validate that file gets successfully moved to the target location
-    Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest"
+    Then Validate output file generated by xmlReader and file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest"
diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java
@@ -120,6 +120,21 @@ public static void validateOutputFileGeneratedByFileSinkPluginIsEqualToExpectedO
     }
   }
 
+  @Then("Validate output file generated by xmlReader and file sink plugin {string} is equal to expected" +
+    " output file {string}")
+  public static void validateOutputFileGeneratedByFileSinkPluginIsEqualToExpectedOutputFileForXml
+    (String fileSinkBucketPath, String expectedOutputFilePath) {
+    if (TestSetupHooks.testOnCdap.equals("true")) {
+      String bucketName = TestSetupHooks.fileSourceBucket;
+      ValidationHelper.compareDataForXml(fileSinkBucketPath,
+                                         PluginPropertyUtils.pluginProp(expectedOutputFilePath), bucketName);
+    } else {
+      String outputFileSinkBucketName = PluginPropertyUtils.pluginProp(fileSinkBucketPath);
+      ValidationHelper.listBucketObjects(outputFileSinkBucketName.substring(5),
+                                         PluginPropertyUtils.pluginProp(expectedOutputFilePath));
+    }
+  }
+
   @Then("Validate output Orc file generated by file sink plugin {string} is equal to expected output file {string}")
   public void validateOutputOrcFileGeneratedByFileSinkPluginIsEqualToExpectedOutputFile(String fileSinkBucketPath,
     String expectedOutputFilePath) throws IOException {

diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java
@@ -472,7 +472,7 @@ private static String createGCSBucketWithFile(String filePath) throws IOExceptio
   }
 
   private static String createGCSBucketWithXmlFile(String filePath) throws IOException, URISyntaxException {
-    String bucketName = StorageClient.createBucket("e2e-test-xml").getName();
+    String bucketName = StorageClient.createBucket("e2e-test-" + UUID.randomUUID()).getName();
     StorageClient.uploadObject(bucketName, filePath, filePath);
     return bucketName;
   }
@@ -522,6 +522,8 @@ public static void createBucketWithXmlFile() throws IOException, URISyntaxExcept
     fileSourceBucket = createGCSBucketWithXmlFile(PluginPropertyUtils.pluginProp("xmlFile"));
     PluginPropertyUtils.addPluginProp("xmlTestFile", "gs://" + fileSourceBucket + "/"  +
       PluginPropertyUtils.pluginProp("xmlFile"));
+    PluginPropertyUtils.addPluginProp("bucketName", fileSourceBucket);
+    PluginPropertyUtils.addPluginProp("targetFolder", "gs://" + fileSourceBucket + "/");
     BeforeActions.scenario.write("xml test bucket name - " + fileSourceBucket);
   }
 

diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java
@@ -97,6 +97,11 @@ public static void compareData(String fileSinkBucketPath, String expectedOutputF
     validateBothFilesData(outputFileData, expectedOutputFilePath);
   }
 
+  public static void compareDataForXml(String fileSinkBucketPath, String expectedOutputFilePath, String bucketName) {
+    List<String> outputFileData = readAllFilesFromDirectory(PluginPropertyUtils.pluginProp(fileSinkBucketPath));
+    validateBothFilesDataForXml(outputFileData, expectedOutputFilePath, bucketName);
+  }
+
   // Reading data from all part-r files present in a directory.
   public static List<String> readAllFilesFromDirectory(String directoryPath) {
     List<String> outputFileData = new ArrayList<>();
@@ -144,4 +149,40 @@ private static boolean compareFiles(String firstFilePath, String secondFilePath)
 
     return Arrays.equals(first, second);
   }
+
+  private static void validateBothFilesDataForXml(List<String> outputFileData, String expectedOutputFilePath,
+                                                  String bucketName) {
+    String[] expectedOutputFileData = fetchObjectData(projectId, expectedOutputFilesBucketName, expectedOutputFilePath);
+    String xmlExpectedFileBucketName = PluginPropertyUtils.pluginProp("xmlBucketName");
+    // Define the bucket names to remove
+    String expectedFileBucketPath = "gs://" + xmlExpectedFileBucketName + "/";
+    String outputFileBucketPath = "gs://" + bucketName + "/";
+
+    // Normalize expected data by removing the bucket name
+    List<String> normalizedExpectedData = new ArrayList<>();
+    for (String line : expectedOutputFileData) {
+      normalizedExpectedData.add(line.replace(expectedFileBucketPath, ""));
+    }
+
+    // Normalize output data by removing the bucket name
+    List<String> normalizedOutputData = new ArrayList<>();
+    for (String line : outputFileData) {
+      normalizedOutputData.add(line.replace(outputFileBucketPath, ""));
+    }
+
+    // Compare normalized data
+    int counter = 0;
+    for (String expectedLine : normalizedExpectedData) {
+      Assert.assertTrue(
+        "Output file and Expected file data should be the same",
+        normalizedOutputData.contains(expectedLine)
+      );
+      counter++;
+    }
+    Assert.assertEquals(
+      "Number of lines in output file and expected file should match",
+      normalizedExpectedData.size(),
+      counter
+    );
+  }
 }
diff --git a/core-plugins/src/e2e-test/resources/pluginParameters.properties b/core-plugins/src/e2e-test/resources/pluginParameters.properties
@@ -304,8 +304,8 @@ xmlFile=testdata/xmldata/testxmlfile.xml
 node=/students/student
 outputForXMLTest=e2e-tests/file/expected_outputs/OUTPUT_FOR_XMLREADER_TEST.csv
 filePattern=^testxmlfile.xml
-folder=gs://e2e-test-xml/
-bucketName=e2e-test-xml
+targetFolder=dummy
+xmlBucketName=e2e-test-xml
 invalidPattern=abcd
 invalidNode=dummy
 xmlFileName=testxmlfile.xml