From 59ac3cc4e944509bb6a19ae08fbdcf4bd3c5d50f Mon Sep 17 00:00:00 2001 From: satyamsufi Date: Tue, 28 Feb 2023 16:00:01 +0530 Subject: [PATCH 1/2] E2E Enhancements --- .github/workflows/e2e.yml | 2 + pom.xml | 12 ++++- .../features/zendesksource/RunTime.feature | 11 ++-- .../zendesksource/RunTimeWithMacros.feature | 8 +-- .../plugin/tests/hooks/TestSetupHooks.java | 37 ++++++++------ .../zendesk/actions/DataValidationHelper.java | 50 +++++++++++++++++++ .../actions/ZendeskPropertiesPageActions.java | 48 +++++++++--------- .../zendesk/stepsdesign/DesignTimeSteps.java | 6 +-- .../resources/pluginParameters.properties | 3 ++ .../expected_outputs/MultiObjects_File1.txt | 9 ---- .../expected_outputs/MultiObjects_File2.txt | 9 ---- .../expected_outputs/SingleObject_Groups.txt | 9 ---- .../batch/ZendeskBatchSourceConfigTest.java | 10 ++-- 13 files changed, 128 insertions(+), 86 deletions(-) create mode 100644 src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java delete mode 100644 src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File1.txt delete mode 100644 src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File2.txt delete mode 100644 src/e2e-test/resources/testdata/expected_outputs/SingleObject_Groups.txt diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5153fd3..5b9b41f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -55,12 +55,14 @@ jobs: secrets: |- ZENDESK_EMAIL:cdapio-github-builds/ZENDESK_EMAIL ZENDESK_API_TOKEN:cdapio-github-builds/ZENDESK_API_TOKEN + ZENDESK_PASSWORD:cdapio-github-builds/ZENDESK_PASSWORD - name: Run tests run: python3 e2e/src/main/scripts/run_e2e_test.py env: ZENDESK_EMAIL: ${{ steps.secrets.outputs.ZENDESK_EMAIL }} ZENDESK_API_TOKEN: ${{ steps.secrets.outputs.ZENDESK_API_TOKEN }} + ZENDESK_PASSWORD: ${{ steps.secrets.outputs.ZENDESK_PASSWORD }} - name: Upload report uses: actions/upload-artifact@v3 diff --git a/pom.xml b/pom.xml index 2f8609c..6a0329e 100644 --- a/pom.xml +++ b/pom.xml @@ -572,6 +572,9 @@ ${ZENDESK_API_TOKEN} + + ${ZENDESK_PASSWORD} + @@ -631,9 +634,16 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.1.0-SNAPSHOT + 0.2.0-SNAPSHOT test + + + io.rest-assured + rest-assured + 5.1.1 + + diff --git a/src/e2e-test/features/zendesksource/RunTime.feature b/src/e2e-test/features/zendesksource/RunTime.feature index 542d935..f220396 100644 --- a/src/e2e-test/features/zendesksource/RunTime.feature +++ b/src/e2e-test/features/zendesksource/RunTime.feature @@ -17,7 +17,7 @@ @Regression Feature: Zendesk Source - Run time scenarios - @TS-ZD-RNTM-01 @BQ_SINK @FILE_PATH @BQ_SINK_CLEANUP + @TS-ZD-RNTM-01 @BQ_SINK @BQ_SINK_CLEANUP @CREATE_GROUP @DELETE_GROUP Scenario: Verify user should be able to preview and deploy the pipeline when plugin is configured for a Non hierarchical object When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" @@ -38,6 +38,7 @@ Feature: Zendesk Source - Run time scenarios And Enter input plugin property: "datasetProject" with value: "datasetprojectId" And Enter input plugin property: "dataset" with value: "dataset" And Enter input plugin property: "table" with value: "bqtarget.table" + And Replace input plugin property: "serviceFilePath" with value: "file.path" And Validate "BigQuery" plugin properties And Close the Plugin Properties page And Preview and run the pipeline @@ -52,9 +53,9 @@ Feature: Zendesk Source - Run time scenarios And Open and capture logs And Verify the pipeline status is "Succeeded" And Close the pipeline logs - Then Validate record created in Sink application for Single object is equal to expected output file "groupsTestOutputFile" + Then Validate record created in Sink application for Single object is equal to expected output file - @TS-ZD-RNTM-02 @BQ_SINK @FILE_PATH @BQ_SINK_CLEANUP + @TS-ZD-RNTM-02 @BQ_SINK @BQ_SINK_CLEANUP @CREATE_GROUP @DELETE_GROUP Scenario: Verify user should be able to preview and deploy the pipeline when plugin is configured for Advanced properties When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" @@ -91,10 +92,10 @@ Feature: Zendesk Source - Run time scenarios And Open and capture logs And Verify the pipeline status is "Succeeded" And Close the pipeline logs - Then Validate record created in Sink application for Single object is equal to expected output file "groupsTestOutputFile" + Then Validate record created in Sink application for Single object is equal to expected output file @TS-ZD-RNTM-03 - Scenario: Verify user should be able to preview and deploy and Run the pipeline when plugin is configured for a herarchical object with File Sink + Scenario: Verify user should be able to preview and deploy and Run the pipeline when plugin is configured for a hierarchical object with File Sink When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" And Navigate to the properties page of plugin: "Zendesk" diff --git a/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature b/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature index 021d37a..1bdd748 100644 --- a/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature +++ b/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature @@ -17,7 +17,7 @@ @Regression Feature: Zendesk Source - Run time scenarios - @TS-ZD-RNTM-MACRO-01 @BQ_SINK @BQ_SINK_CLEANUP @FILE_PATH + @TS-ZD-RNTM-MACRO-01 @BQ_SINK @BQ_SINK_CLEANUP @TEST_DATA @DELETE_TEST_DATA Scenario: Verify user should be able to preview and deploy the pipeline when plugin is configured for a Non hierarchical Object with macros When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" @@ -69,9 +69,9 @@ Feature: Zendesk Source - Run time scenarios And Open and capture logs And Verify the pipeline status is "Succeeded" And Close the pipeline logs - Then Validate record created in Sink application for Single object is equal to expected output file "groupsTestOutputFile" + Then Validate record created in Sink application for Single object is equal to expected output file - @TS-ZD-RNTM-MACRO-02 @BQ_SINK @BQ_SINK_CLEANUP @FILE_PATH + @TS-ZD-RNTM-MACRO-02 @BQ_SINK @BQ_SINK_CLEANUP @TEST_DATA @DELETE_TEST_DATA Scenario: Verify user should be able to preview and deploy the pipeline when plugin is configured for Advanced Properties with macros When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" @@ -116,7 +116,7 @@ Feature: Zendesk Source - Run time scenarios And Open and capture logs And Verify the pipeline status is "Succeeded" And Close the pipeline logs - Then Validate record created in Sink application for Single object is equal to expected output file "groupsTestOutputFile" + Then Validate record created in Sink application for Single object is equal to expected output file @TS-ZD-RNTM-MACRO-03 @BQ_SINK @BQ_SINK_CLEANUP Scenario: Verify pipeline failure message in logs when user provides invalid Credentials with Macros diff --git a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java index c40f445..fca4615 100644 --- a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java @@ -18,6 +18,7 @@ import com.google.cloud.bigquery.BigQueryException; import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.plugin.zendesk.actions.DataValidationHelper; import io.cucumber.java.After; import io.cucumber.java.Before; import org.apache.commons.lang3.RandomStringUtils; @@ -26,12 +27,27 @@ import stepsdesign.BeforeActions; import java.io.IOException; import java.nio.file.Paths; +import java.util.Base64; /** * Represents Test Setup and Clean up hooks. */ public class TestSetupHooks { - private static boolean firstFileSinkTestFlag = true; + public static String testdata_Group = ""; + public static String cred = ""; + + + @Before(order = 1, value = "@CREATE_GROUP") + public void createGroup() { + Base64.Encoder encoder = Base64.getUrlEncoder(); + String email = System.getenv("ZENDESK_EMAIL"); + String password = System.getenv("ZENDESK_PASSWORD"); + String auth = email + ":" + password; + String encodedAuth = encoder.encodeToString(auth.getBytes()); + cred = "Basic " + encodedAuth; + String jsonBody = "{\"group\": {\"name\": \"My Group" + RandomStringUtils.randomAlphanumeric(10) + "\"}}"; + testdata_Group = DataValidationHelper.createGroup(cred, jsonBody); + } @Before(order = 1, value = "@BQ_SINK") public void setTempTargetBQDataset() { @@ -77,19 +93,8 @@ public void deleteMultiSourceTargetBQTable() throws IOException, InterruptedExce } } } - - @Before(order = 1, value = "@FILE_PATH") - public static void setFileAbsolutePath() { - - if (firstFileSinkTestFlag) { - PluginPropertyUtils.addPluginProp("groupsTestOutputFile", Paths.get(TestSetupHooks.class.getResource - ("/" + PluginPropertyUtils.pluginProp("groupsTestOutputFile")).getPath()).toString()); - PluginPropertyUtils.addPluginProp("multiObjectsOutputFile", Paths.get(TestSetupHooks.class.getResource - ("/" + PluginPropertyUtils.pluginProp("multiObjectsOutputFile")).getPath()).toString()); - PluginPropertyUtils.addPluginProp("multiObjectsOutputFile1", Paths.get(TestSetupHooks.class.getResource - ("/" + PluginPropertyUtils.pluginProp("multiObjectsOutputFile1")).getPath()).toString()); - - firstFileSinkTestFlag = false; - } + @After(order = 2, value = "@DELETE_GROUP") + public void deleteGroup() { + DataValidationHelper.deleteGroup(cred); + } } -} diff --git a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java new file mode 100644 index 0000000..a2ee62b --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.zendesk.actions; + +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import io.restassured.response.ResponseBody; +import static io.restassured.RestAssured.given; + + +/** + * Zendesk utility - enhancements. + */ +public class DataValidationHelper { + private static String baseURI = "https://cloudsufi.zendesk.com/api/v2"; + public static String createGroup(String cred, String jsonBody) { + Response response = given() + .header("authorization", cred) + .accept(ContentType.JSON) + .contentType(ContentType.JSON) + .and() + .body(jsonBody) + .when() + .post(baseURI + "/groups.json") + .then().extract().response(); + + ResponseBody responseBody = response; + + return responseBody.asString(); + } + public static void deleteGroup(String cred) { + Response response1 = given() + .header("authorization", cred) + .delete(baseURI + "/groups/" + ZendeskPropertiesPageActions.uniqueId + ".json"); + } + } diff --git a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java index 3f79b85..f736e75 100644 --- a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java +++ b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java @@ -26,6 +26,7 @@ import io.cdap.e2e.utils.ElementHelper; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.e2e.utils.SeleniumHelper; +import io.cdap.plugin.tests.hooks.TestSetupHooks; import io.cdap.plugin.utils.enums.Subdomains; import io.cdap.plugin.zendesk.locators.ZendeskPropertiesPage; import org.junit.Assert; @@ -49,6 +50,8 @@ public class ZendeskPropertiesPageActions { private static Gson gson = new Gson(); private static List bigQueryrows = new ArrayList<>(); + public static BigInteger uniqueId; + static { SeleniumHelper.getPropertiesLocators(ZendeskPropertiesPage.class); } @@ -91,31 +94,21 @@ public static void selectDropdowWithMultipleOptionsForObjectsToSkip(List ElementHelper.clickUsingActions(CdfPluginPropertiesLocators.pluginPropertiesPageHeader); } - public static void verifyIfRecordCreatedInSinkForSingleObjectIsCorrect(String expectedOutputFile) + public static void verifyIfRecordCreatedInSinkForSingleObjectIsCorrect() throws IOException, InterruptedException { - List expectedOutput = new ArrayList<>(); - try (BufferedReader bf1 = Files.newBufferedReader(Paths.get(PluginPropertyUtils.pluginProp(expectedOutputFile)))) { - String line; - while ((line = bf1.readLine()) != null) { - expectedOutput.add(line); - } - } - - for (int expectedRow = 0; expectedRow < expectedOutput.size(); expectedRow++) { - JsonObject expectedOutputAsJson = gson.fromJson(expectedOutput.get(expectedRow), JsonObject.class); - BigInteger uniqueId = expectedOutputAsJson.get("id").getAsBigInteger(); + JsonObject expectedOutputAsJson = gson.fromJson(TestSetupHooks.testdata_Group, JsonObject.class); + JsonObject group = (JsonObject) expectedOutputAsJson.get("group"); + System.out.println(group); + uniqueId = group.get("id").getAsBigInteger(); getBigQueryTableData(PluginPropertyUtils.pluginProp("dataset"), PluginPropertyUtils.pluginProp("bqtarget.table"), uniqueId); - - } - for (int row = 0; row < bigQueryrows.size() && row < expectedOutput.size(); row++) { - Assert.assertTrue(ZendeskPropertiesPageActions.compareValueOfBothResponses(expectedOutput.get(row), - bigQueryrows.get(row))); + Assert.assertTrue(ZendeskPropertiesPageActions.compareValueOfBothResponses(group.toString(), + bigQueryrows.get(0))); } - } + public static void verifyIfRecordCreatedInSinkForMultipleObjectsAreCorrect(String expectedOutputFile) - throws IOException, InterruptedException { + throws IOException, InterruptedException { List expectedOutput = new ArrayList<>(); try (BufferedReader bf1 = Files.newBufferedReader(Paths.get(PluginPropertyUtils.pluginProp(expectedOutputFile)))) { String line; @@ -126,7 +119,7 @@ public static void verifyIfRecordCreatedInSinkForMultipleObjectsAreCorrect(Strin List bigQueryDatasetTables = new ArrayList<>(); TableResult tablesSchema = ZendeskPropertiesPageActions.getTableNamesFromDataSet - (PluginPropertyUtils.pluginProp("dataset")); + (PluginPropertyUtils.pluginProp("dataset")); tablesSchema.iterateAll().forEach(value -> bigQueryDatasetTables.add(value.get(0).getValue().toString())); System.out.println(bigQueryDatasetTables.size()); @@ -134,13 +127,14 @@ public static void verifyIfRecordCreatedInSinkForMultipleObjectsAreCorrect(Strin JsonObject expectedOutputAsJson = gson.fromJson(expectedOutput.get(expectedRow), JsonObject.class); BigInteger uniqueId = expectedOutputAsJson.get("id").getAsBigInteger(); getBigQueryTableData(PluginPropertyUtils.pluginProp("dataset"), - bigQueryDatasetTables.get(0), uniqueId); + bigQueryDatasetTables.get(0), uniqueId); } for (int row = 0; row < bigQueryrows.size() && row < expectedOutput.size(); row++) { Assert.assertTrue(ZendeskPropertiesPageActions.compareValueOfBothResponses(expectedOutput.get(row), - bigQueryrows.get(row))); + bigQueryrows.get(row))); + } } - } + static boolean compareValueOfBothResponses(String zendeskResponse, String bigQueryResponse) { Type type = new TypeToken>() { @@ -149,13 +143,17 @@ static boolean compareValueOfBothResponses(String zendeskResponse, String bigQue Map bigQueryResponseInMap = gson.fromJson(bigQueryResponse, type); MapDifference mapDifference = Maps.difference(zendeskResponseInmap, bigQueryResponseInMap); logger.info("Assertion :" + mapDifference); - return mapDifference.areEqual(); } public static void getBigQueryTableData(String dataset, String table, BigInteger uniqueId) throws IOException, InterruptedException { String projectId = PluginPropertyUtils.pluginProp("projectId"); + + // Altering the table to get the field names in sync with Zendesk response. + String alterQuery = "ALTER TABLE `" + projectId + "." + dataset + "." + table + + "` RENAME COLUMN updatedAt to updated_at,RENAME COLUMN createdAt to created_at "; + BigQueryClient.executeQuery(alterQuery); String selectQuery = "SELECT TO_JSON(t) FROM `" + projectId + "." + dataset + "." + table + "` AS t WHERE " + "id=" + uniqueId + " "; TableResult result = BigQueryClient.getQueryResult(selectQuery); @@ -169,5 +167,5 @@ public static TableResult getTableNamesFromDataSet(String bqTargetDataset) throw return BigQueryClient.getQueryResult(selectQuery); } -} + } diff --git a/src/e2e-test/java/io/cdap/plugin/zendesk/stepsdesign/DesignTimeSteps.java b/src/e2e-test/java/io/cdap/plugin/zendesk/stepsdesign/DesignTimeSteps.java index aa2955e..8e0ec0e 100644 --- a/src/e2e-test/java/io/cdap/plugin/zendesk/stepsdesign/DesignTimeSteps.java +++ b/src/e2e-test/java/io/cdap/plugin/zendesk/stepsdesign/DesignTimeSteps.java @@ -44,10 +44,10 @@ public void fillObjectsToSkipListWithBelowListedObjects(DataTable table) { ZendeskPropertiesPageActions.selectDropdowWithMultipleOptionsForObjectsToSkip(tablesList); } - @Then("Validate record created in Sink application for Single object is equal to expected output file {string}") - public void verifyIfNewRecordCreatedInSinkApplicationForObjectIsCorrect(String expectedOutputFile) + @Then("Validate record created in Sink application for Single object is equal to expected output file") + public void verifyIfNewRecordCreatedInSinkApplicationForObjectIsCorrect() throws IOException, InterruptedException { - ZendeskPropertiesPageActions.verifyIfRecordCreatedInSinkForSingleObjectIsCorrect(expectedOutputFile); + ZendeskPropertiesPageActions.verifyIfRecordCreatedInSinkForSingleObjectIsCorrect(); } @Then("Validate record created in Sink application for Multi object is equal to expected output file {string}") diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 53fa864..d13ea1b 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,6 +1,7 @@ #Credentialls admin.email = ZENDESK_EMAIL admin.apitoken = ZENDESK_API_TOKEN +password = ZENDESK_PASSWORD #Valid Porperties admin.subdomain = cloudsufi @@ -28,6 +29,7 @@ projectId=cdf-athena datasetprojectId=cdf-athena dataset=enterprise_test_automation bqtarget.table=target-table +file.path=/Users/tiwarisatyam/Downloads/cdf-athena-48053fab02c2.json groups=Groups ticket_comments=TicketComments @@ -170,3 +172,4 @@ schema.objecttopull.satisfactionrating = [{"key":"id","value":"long"},\ {"key":"createdAt","value":"string"},{"key":"updatedAt","value":"string"},\ {"key":"comment","value":"string"},{"key":"reason","value":"string"},\ {"key":"reasonId","value":"long"},{"key":"reasonCode","value":"long"}] + diff --git a/src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File1.txt b/src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File1.txt deleted file mode 100644 index 6008373..0000000 --- a/src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File1.txt +++ /dev/null @@ -1,9 +0,0 @@ -{"createdAt":"2022-06-28T07:02:08Z","default":false,"deleted":false,"description":"","id":5702248917277,"is_public":true,"name":"june28_1","updatedAt":"2022-06-28T07:02:08Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702248917277.json"} -{"createdAt":"2022-06-28T07:02:16Z","default":false,"deleted":false,"description":"","id":5702220951197,"is_public":true,"name":"june28_2","updatedAt":"2022-06-28T07:02:16Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702220951197.json"} -{"createdAt":"2022-06-28T07:02:22Z","default":false,"deleted":false,"description":"","id":5702237944477,"is_public":true,"name":"june28_3","updatedAt":"2022-06-28T07:02:22Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702237944477.json"} -{"createdAt":"2022-06-28T07:02:31Z","default":false,"deleted":false,"description":"","id":5702238282397,"is_public":true,"name":"june28_4","updatedAt":"2022-06-28T07:02:31Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702238282397.json"} -{"createdAt":"2022-05-25T04:18:59Z","default":true,"deleted":false,"description":"","id":5281674907293,"is_public":true,"name":"Support","updatedAt":"2022-05-25T04:18:59Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5281674907293.json"} -{"createdAt":"2022-08-15T13:40:33Z","default":false,"deleted":false,"description":"","id":6328289794973,"is_public":true,"name":"Test Group","updatedAt":"2022-08-15T13:40:33Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/6328289794973.json"} -{"createdAt":"2022-06-08T09:58:03Z","default":false,"deleted":false,"description":"Sample Group","id":5446908715549,"is_public":true,"name":"Test_Group_VK","updatedAt":"2022-06-08T09:58:03Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5446908715549.json"} -{"createdAt":"2022-06-23T10:57:11Z","default":false,"deleted":false,"description":"regressionv123","id":5645781946141,"is_public":true,"name":"june23","updatedAt":"2022-06-23T10:57:11Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5645781946141.json"} -{"createdAt":"2022-06-20T10:40:30Z","default":false,"deleted":false,"description":"testing in june","id":5602787968413,"is_public":true,"name":"June_Groups","updatedAt":"2022-06-20T10:40:30Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5602787968413.json"} diff --git a/src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File2.txt b/src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File2.txt deleted file mode 100644 index 2776740..0000000 --- a/src/e2e-test/resources/testdata/expected_outputs/MultiObjects_File2.txt +++ /dev/null @@ -1,9 +0,0 @@ -{"createdAt":"2022-06-28T07:02:08Z","default":false,"deleted":false,"description":"","id":5702248917277,"is_public":true,"name":"june28_1","updatedAt":"2022-06-28T07:02:08Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702248917277.json"} -{"createdAt":"2022-06-28T07:02:16Z","default":false,"deleted":false,"description":"","id":5702220951197,"is_public":true,"name":"june28_2","updatedAt":"2022-06-28T07:02:16Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702220951197.json"} -{"createdAt":"2022-06-28T07:02:22Z","default":false,"deleted":false,"description":"","id":5702237944477,"is_public":true,"name":"june28_3","updatedAt":"2022-06-28T07:02:22Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702237944477.json"} -{"createdAt":"2022-06-28T07:02:31Z","default":false,"deleted":false,"description":"","id":5702238282397,"is_public":true,"name":"june28_4","updatedAt":"2022-06-28T07:02:31Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702238282397.json"} -{"createdAt":"2022-05-25T04:18:59Z","default":true,"deleted":false,"description":"","id":5281674907293,"is_public":true,"name":"Support","updatedAt":"2022-05-25T04:18:59Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5281674907293.json"} -{"createdAt":"2022-08-15T13:40:33Z","default":false,"deleted":false,"description":"","id":6328289794973,"is_public":true,"name":"Test Group","updatedAt":"2022-08-15T13:40:33Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/6328289794973.json"} -{"createdAt":"2022-06-08T09:58:03Z","default":false,"deleted":false,"description":"Sample Group","id":5446908715549,"is_public":true,"name":"Test_Group_VK","updatedAt":"2022-06-08T09:58:03Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5446908715549.json"} -{"createdAt":"2022-06-23T10:57:11Z","default":false,"deleted":false,"description":"regressionv123","id":5645781946141,"is_public":true,"name":"june23","updatedAt":"2022-06-23T10:57:11Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5645781946141.json"} -{"createdAt":"2022-06-20T10:40:30Z","default":false,"deleted":false,"description":"testing in june","id":5602787968413,"is_public":true,"name":"June_Groups","updatedAt":"2022-06-20T10:40:30Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5602787968413.json"} \ No newline at end of file diff --git a/src/e2e-test/resources/testdata/expected_outputs/SingleObject_Groups.txt b/src/e2e-test/resources/testdata/expected_outputs/SingleObject_Groups.txt deleted file mode 100644 index 2776740..0000000 --- a/src/e2e-test/resources/testdata/expected_outputs/SingleObject_Groups.txt +++ /dev/null @@ -1,9 +0,0 @@ -{"createdAt":"2022-06-28T07:02:08Z","default":false,"deleted":false,"description":"","id":5702248917277,"is_public":true,"name":"june28_1","updatedAt":"2022-06-28T07:02:08Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702248917277.json"} -{"createdAt":"2022-06-28T07:02:16Z","default":false,"deleted":false,"description":"","id":5702220951197,"is_public":true,"name":"june28_2","updatedAt":"2022-06-28T07:02:16Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702220951197.json"} -{"createdAt":"2022-06-28T07:02:22Z","default":false,"deleted":false,"description":"","id":5702237944477,"is_public":true,"name":"june28_3","updatedAt":"2022-06-28T07:02:22Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702237944477.json"} -{"createdAt":"2022-06-28T07:02:31Z","default":false,"deleted":false,"description":"","id":5702238282397,"is_public":true,"name":"june28_4","updatedAt":"2022-06-28T07:02:31Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5702238282397.json"} -{"createdAt":"2022-05-25T04:18:59Z","default":true,"deleted":false,"description":"","id":5281674907293,"is_public":true,"name":"Support","updatedAt":"2022-05-25T04:18:59Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5281674907293.json"} -{"createdAt":"2022-08-15T13:40:33Z","default":false,"deleted":false,"description":"","id":6328289794973,"is_public":true,"name":"Test Group","updatedAt":"2022-08-15T13:40:33Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/6328289794973.json"} -{"createdAt":"2022-06-08T09:58:03Z","default":false,"deleted":false,"description":"Sample Group","id":5446908715549,"is_public":true,"name":"Test_Group_VK","updatedAt":"2022-06-08T09:58:03Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5446908715549.json"} -{"createdAt":"2022-06-23T10:57:11Z","default":false,"deleted":false,"description":"regressionv123","id":5645781946141,"is_public":true,"name":"june23","updatedAt":"2022-06-23T10:57:11Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5645781946141.json"} -{"createdAt":"2022-06-20T10:40:30Z","default":false,"deleted":false,"description":"testing in june","id":5602787968413,"is_public":true,"name":"June_Groups","updatedAt":"2022-06-20T10:40:30Z","url":"https://cloudsufi.zendesk.com/api/v2/groups/5602787968413.json"} \ No newline at end of file diff --git a/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java b/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java index b30eff6..89dae30 100644 --- a/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java @@ -41,7 +41,7 @@ public class ZendeskBatchSourceConfigTest { private static final String MOCK_STAGE = "mockStage"; - private static MockPipelineConfigurer pipelineConfigurer = null; + private static MockPipelineConfigurer pipelineConfigure = null; @Test public void testValidate() { @@ -352,8 +352,8 @@ public void testConfigurePipelineWithInvalidBasicParam() { ""); try { ZendeskBatchSource zendeskBatchSource = new ZendeskBatchSource(config); - pipelineConfigurer = new MockPipelineConfigurer(null); - zendeskBatchSource.configurePipeline(pipelineConfigurer); + pipelineConfigure = new MockPipelineConfigurer(null); + zendeskBatchSource.configurePipeline(pipelineConfigure); Assert.fail("Reference name validation should fail"); } catch (ValidationException ve) { List failures = ve.getFailures(); @@ -414,8 +414,8 @@ public void testConfigurePipelineMultiSourceWithInvalidBasicParam() { ""); try { ZendeskBatchMultiSource zendeskBatchMultiSource = new ZendeskBatchMultiSource(config); - pipelineConfigurer = new MockPipelineConfigurer(null); - zendeskBatchMultiSource.configurePipeline(pipelineConfigurer); + pipelineConfigure = new MockPipelineConfigurer(null); + zendeskBatchMultiSource.configurePipeline(pipelineConfigure); Assert.fail("Reference name validation should fail"); } catch (ValidationException ve) { List failures = ve.getFailures(); From 38336536b330483848f56e388de59b6e2cf3e39f Mon Sep 17 00:00:00 2001 From: satyamsufi Date: Thu, 2 Mar 2023 12:21:41 +0530 Subject: [PATCH 2/2] Re-work --- .../features/zendesksource/RunTimeWithMacros.feature | 4 ++-- .../io/cdap/plugin/tests/hooks/TestSetupHooks.java | 12 ++++++------ .../plugin/zendesk/actions/DataValidationHelper.java | 5 +++-- .../actions/ZendeskPropertiesPageActions.java | 10 +++------- src/e2e-test/resources/pluginParameters.properties | 3 +++ .../source/batch/ZendeskBatchSourceConfigTest.java | 10 +++++----- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature b/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature index 1bdd748..97fbc58 100644 --- a/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature +++ b/src/e2e-test/features/zendesksource/RunTimeWithMacros.feature @@ -17,7 +17,7 @@ @Regression Feature: Zendesk Source - Run time scenarios - @TS-ZD-RNTM-MACRO-01 @BQ_SINK @BQ_SINK_CLEANUP @TEST_DATA @DELETE_TEST_DATA + @TS-ZD-RNTM-MACRO-01 @BQ_SINK @BQ_SINK_CLEANUP @CREATE_GROUP @DELETE_GROUP Scenario: Verify user should be able to preview and deploy the pipeline when plugin is configured for a Non hierarchical Object with macros When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" @@ -71,7 +71,7 @@ Feature: Zendesk Source - Run time scenarios And Close the pipeline logs Then Validate record created in Sink application for Single object is equal to expected output file - @TS-ZD-RNTM-MACRO-02 @BQ_SINK @BQ_SINK_CLEANUP @TEST_DATA @DELETE_TEST_DATA + @TS-ZD-RNTM-MACRO-02 @BQ_SINK @BQ_SINK_CLEANUP @CREATE_GROUP @DELETE_GROUP Scenario: Verify user should be able to preview and deploy the pipeline when plugin is configured for Advanced Properties with macros When Open Datafusion Project to configure pipeline And Select plugin: "Zendesk" from the plugins list as: "Source" diff --git a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java index fca4615..5507c3e 100644 --- a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java @@ -33,8 +33,8 @@ * Represents Test Setup and Clean up hooks. */ public class TestSetupHooks { - public static String testdata_Group = ""; - public static String cred = ""; + public static String groupResponse = ""; + public static String credentials = ""; @Before(order = 1, value = "@CREATE_GROUP") @@ -44,9 +44,9 @@ public void createGroup() { String password = System.getenv("ZENDESK_PASSWORD"); String auth = email + ":" + password; String encodedAuth = encoder.encodeToString(auth.getBytes()); - cred = "Basic " + encodedAuth; + credentials = "Basic " + encodedAuth; String jsonBody = "{\"group\": {\"name\": \"My Group" + RandomStringUtils.randomAlphanumeric(10) + "\"}}"; - testdata_Group = DataValidationHelper.createGroup(cred, jsonBody); + groupResponse = DataValidationHelper.createGroup(credentials, jsonBody); } @Before(order = 1, value = "@BQ_SINK") @@ -95,6 +95,6 @@ public void deleteMultiSourceTargetBQTable() throws IOException, InterruptedExce } @After(order = 2, value = "@DELETE_GROUP") public void deleteGroup() { - DataValidationHelper.deleteGroup(cred); - } + DataValidationHelper.deleteGroup(credentials); } +} diff --git a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java index a2ee62b..b908d33 100644 --- a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java +++ b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/DataValidationHelper.java @@ -16,6 +16,7 @@ package io.cdap.plugin.zendesk.actions; +import io.cdap.e2e.utils.PluginPropertyUtils; import io.restassured.http.ContentType; import io.restassured.response.Response; import io.restassured.response.ResponseBody; @@ -26,7 +27,7 @@ * Zendesk utility - enhancements. */ public class DataValidationHelper { - private static String baseURI = "https://cloudsufi.zendesk.com/api/v2"; + private static String baseURI = PluginPropertyUtils.pluginProp("base.URI"); public static String createGroup(String cred, String jsonBody) { Response response = given() .header("authorization", cred) @@ -47,4 +48,4 @@ public static void deleteGroup(String cred) { .header("authorization", cred) .delete(baseURI + "/groups/" + ZendeskPropertiesPageActions.uniqueId + ".json"); } - } +} diff --git a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java index f736e75..00a3249 100644 --- a/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java +++ b/src/e2e-test/java/io/cdap/plugin/zendesk/actions/ZendeskPropertiesPageActions.java @@ -96,16 +96,14 @@ public static void selectDropdowWithMultipleOptionsForObjectsToSkip(List public static void verifyIfRecordCreatedInSinkForSingleObjectIsCorrect() throws IOException, InterruptedException { - JsonObject expectedOutputAsJson = gson.fromJson(TestSetupHooks.testdata_Group, JsonObject.class); + JsonObject expectedOutputAsJson = gson.fromJson(TestSetupHooks.groupResponse, JsonObject.class); JsonObject group = (JsonObject) expectedOutputAsJson.get("group"); - System.out.println(group); uniqueId = group.get("id").getAsBigInteger(); getBigQueryTableData(PluginPropertyUtils.pluginProp("dataset"), PluginPropertyUtils.pluginProp("bqtarget.table"), uniqueId); Assert.assertTrue(ZendeskPropertiesPageActions.compareValueOfBothResponses(group.toString(), bigQueryrows.get(0))); - } - + } public static void verifyIfRecordCreatedInSinkForMultipleObjectsAreCorrect(String expectedOutputFile) throws IOException, InterruptedException { @@ -116,12 +114,10 @@ public static void verifyIfRecordCreatedInSinkForMultipleObjectsAreCorrect(Strin expectedOutput.add(line); } } - List bigQueryDatasetTables = new ArrayList<>(); TableResult tablesSchema = ZendeskPropertiesPageActions.getTableNamesFromDataSet (PluginPropertyUtils.pluginProp("dataset")); tablesSchema.iterateAll().forEach(value -> bigQueryDatasetTables.add(value.get(0).getValue().toString())); - System.out.println(bigQueryDatasetTables.size()); for (int expectedRow = 0; expectedRow < expectedOutput.size(); expectedRow++) { JsonObject expectedOutputAsJson = gson.fromJson(expectedOutput.get(expectedRow), JsonObject.class); @@ -133,7 +129,7 @@ public static void verifyIfRecordCreatedInSinkForMultipleObjectsAreCorrect(Strin Assert.assertTrue(ZendeskPropertiesPageActions.compareValueOfBothResponses(expectedOutput.get(row), bigQueryrows.get(row))); } - } + } static boolean compareValueOfBothResponses(String zendeskResponse, String bigQueryResponse) { diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index d13ea1b..25a48c2 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -3,6 +3,9 @@ admin.email = ZENDESK_EMAIL admin.apitoken = ZENDESK_API_TOKEN password = ZENDESK_PASSWORD +#API +base.URI = https://cloudsufi.zendesk.com/api/v2 + #Valid Porperties admin.subdomain = cloudsufi objectstopull.group=Groups diff --git a/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java b/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java index 89dae30..b30eff6 100644 --- a/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/zendesk/source/batch/ZendeskBatchSourceConfigTest.java @@ -41,7 +41,7 @@ public class ZendeskBatchSourceConfigTest { private static final String MOCK_STAGE = "mockStage"; - private static MockPipelineConfigurer pipelineConfigure = null; + private static MockPipelineConfigurer pipelineConfigurer = null; @Test public void testValidate() { @@ -352,8 +352,8 @@ public void testConfigurePipelineWithInvalidBasicParam() { ""); try { ZendeskBatchSource zendeskBatchSource = new ZendeskBatchSource(config); - pipelineConfigure = new MockPipelineConfigurer(null); - zendeskBatchSource.configurePipeline(pipelineConfigure); + pipelineConfigurer = new MockPipelineConfigurer(null); + zendeskBatchSource.configurePipeline(pipelineConfigurer); Assert.fail("Reference name validation should fail"); } catch (ValidationException ve) { List failures = ve.getFailures(); @@ -414,8 +414,8 @@ public void testConfigurePipelineMultiSourceWithInvalidBasicParam() { ""); try { ZendeskBatchMultiSource zendeskBatchMultiSource = new ZendeskBatchMultiSource(config); - pipelineConfigure = new MockPipelineConfigurer(null); - zendeskBatchMultiSource.configurePipeline(pipelineConfigure); + pipelineConfigurer = new MockPipelineConfigurer(null); + zendeskBatchMultiSource.configurePipeline(pipelineConfigurer); Assert.fail("Reference name validation should fail"); } catch (ValidationException ve) { List failures = ve.getFailures();