diff --git a/pom.xml b/pom.xml index 81f5b24..d25eadc 100644 --- a/pom.xml +++ b/pom.xml @@ -26,8 +26,10 @@ UTF-8 - 6.1.2 + 6.8.0-SNAPSHOT 2.0.2 + 6.4.0 + ${project.basedir}/src/test/java/ @@ -46,11 +48,22 @@ io.cdap.cdap cdap-etl-api ${cdap.version} + + + guava + com.google.guava + + + + + com.google.guava + guava + 30.1.1-jre io.cdap.cdap cdap-data-pipeline - ${cdap.version} + ${cdap.data.pipeline.version} test @@ -68,6 +81,7 @@ + ${testSourceLocation} org.apache.maven.plugins @@ -146,5 +160,110 @@ - + + + e2e-tests + + src/e2e-test/java + + + + + src/e2e-test/resources + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + true + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M5 + + + TestRunner.java + + + classes + 2 + 2 + true + + + + ${GOOGLE_APPLICATION_CREDENTIALS} + + + ${SERVICE_ACCOUNT_TYPE} + + + ${SERVICE_ACCOUNT_FILE_PATH} + + + ${SERVICE_ACCOUNT_JSON} + + + + + + + integration-test + + + + + + + net.masterthought + maven-cucumber-reporting + 5.5.0 + + + + execution + verify + + generate + + + Cucumber Reports + target/cucumber-reports/advanced-reports + 1 + false + ${project.build.directory}/cucumber-reports + + **/*.json + + ${project.build.directory}/cucumber-reports + true + + + + + + + + + io.cdap.tests.e2e + cdap-e2e-framework + 0.0.1-SNAPSHOT + test + + + ch.qos.logback + logback-classic + 1.2.8 + runtime + + + + + diff --git a/src/e2e-test/features/addfield/ErrorScenariosAddField.feature b/src/e2e-test/features/addfield/ErrorScenariosAddField.feature new file mode 100644 index 0000000..8957ad7 --- /dev/null +++ b/src/e2e-test/features/addfield/ErrorScenariosAddField.feature @@ -0,0 +1,45 @@ +@Add_Field +Feature: AddField Plugin - Verify error scenarios + + @ADD_FIELD-01 + Scenario: Verify add field validation errors for mandatory fields + Given Open Datafusion Project to configure pipeline + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Navigate to the properties page of plugin: "AddField" + And Click on the Validate button + And Verify mandatory property error for below listed properties: + | fieldName | + + @ADD_FIELD-02 + Scenario: Validate invalid error messages in add field plugin without any input data + Given Open Datafusion Project to configure pipeline + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Navigate to the properties page of plugin: "AddField" + And Enter input plugin property: "addFieldFieldName" with value: "afFieldName" + And Click on the Get Schema button + And Verify that the Plugin Property: "fieldValue" is displaying an in-line error message: "addFieldErrorMessageInvalidField" + And Verify that the Plugin Property: "asUUID" is displaying an in-line error message: "addFieldErrorMessageInvalidField" + + @ADD_FIELD-03 + Scenario: Verify error count for add field plugin for mandatory fields + Given Open Datafusion Project to configure pipeline + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Navigate to the properties page of plugin: "AddField" + And Click on the Validate button + And Verify plugin properties validation fails with 1 error + + @ADD_FIELD-04 + Scenario: Validate errors when field value is given and generate uuid as value is set to true + Given Open Datafusion Project to configure pipeline + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Navigate to the properties page of plugin: "AddField" + And Enter input plugin property: "addFieldFieldName" with value: "afFieldName" + And Enter input plugin property: "addFieldFieldValue" with value: "afFieldValue" + And Select dropdown plugin property: "addFieldGenerateUUID" with option value: "true" + And Click on the Get Schema button + And Verify that the Plugin Property: "fieldValue" is displaying an in-line error message: "addFieldErrorMessageValidFieldValue" + And Verify that the Plugin Property: "asUUID" is displaying an in-line error message: "addFieldErrorMessageValidFieldValue" diff --git a/src/e2e-test/features/addfield/MacroScenarioAddField.feature b/src/e2e-test/features/addfield/MacroScenarioAddField.feature new file mode 100644 index 0000000..f05e7ea --- /dev/null +++ b/src/e2e-test/features/addfield/MacroScenarioAddField.feature @@ -0,0 +1,58 @@ +@Add_Field +Feature: AddField Plugin - Verify macro scenarios + + @FILE_SOURCE_TEST @BQ_SINK_TEST + Scenario: Verify add field functionality with macro arguments using File to BigQuery pipeline + Given Open Datafusion Project to configure pipeline + And Select plugin: "File" from the plugins list as: "Source" + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Expand Plugin group in the LHS plugins list: "Sink" + And Select plugin: "BigQuery" from the plugins list as: "Sink" + And Connect plugins: "File" and "AddField" to establish connection + And Connect plugins: "AddField" and "BigQuery" to establish connection + And Navigate to the properties page of plugin: "File" + And Enter input plugin property: "referenceName" with value: "FileReferenceName" + And Enter input plugin property: "path" with value: "csvAllDataTypeFile" + And Select dropdown plugin property: "format" with option value: "csv" + And Click plugin property: "skipHeader" + And Click on the Get Schema button + And Validate "File" plugin properties + And Close the Plugin Properties page + And Navigate to the properties page of plugin: "AddField" + And Enter input plugin property: "addFieldFieldName" with value: "afFieldName" + And Click on the Macro button of Property: "addFieldFieldValue" and set the value to: "fieldvalue" + And Select dropdown plugin property: "addFieldGenerateUUID" with option value: "false" + And Validate "AddField" plugin properties + And Validate output schema with expectedSchema "csvAllDataTypeFileSchemaAddField" + And Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Replace input plugin property: "projectId" with value: "projectId" + And Enter input plugin property: "datasetProjectId" with value: "projectId" + Then Override Service account details if set in environment variables + And Enter input plugin property: "referenceName" with value: "BQReferenceName" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqTargetTable" + And Click plugin property: "truncateTable" + And Click plugin property: "updateTableSchema" + And Validate "BigQuery" plugin properties + And Close the Plugin Properties page + And Save the pipeline + And Preview and run the pipeline + And Enter runtime argument value "afFieldValue" for key "fieldvalue" + And Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + And Enter runtime argument value "afFieldValue" for key "fieldvalue" + And Run the Pipeline in Runtime with runtime arguments + And Wait till pipeline is in running state + And Open and capture logs + And Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate OUT record count is equal to IN record count + Then Verify column: "afFieldName" is added in target BigQuery table: "bqTargetTable" diff --git a/src/e2e-test/features/addfield/RunTimeScenarioAddField.feature b/src/e2e-test/features/addfield/RunTimeScenarioAddField.feature new file mode 100644 index 0000000..1094f5e --- /dev/null +++ b/src/e2e-test/features/addfield/RunTimeScenarioAddField.feature @@ -0,0 +1,102 @@ +@Add_Field +Feature: AddField Plugin - Run time scenarios + + @BQ_SINK_TEST @FILE_SOURCE_TEST + Scenario: Verify add field plugin functionality by setting field value using File to BigQuery pipeline + Given Open Datafusion Project to configure pipeline + And Select plugin: "File" from the plugins list as: "Source" + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Expand Plugin group in the LHS plugins list: "Sink" + And Select plugin: "BigQuery" from the plugins list as: "Sink" + And Connect plugins: "File" and "AddField" to establish connection + And Connect plugins: "AddField" and "BigQuery" to establish connection + And Navigate to the properties page of plugin: "File" + And Enter input plugin property: "referenceName" with value: "FileReferenceName" + And Enter input plugin property: "path" with value: "csvAllDataTypeFile" + And Select dropdown plugin property: "format" with option value: "csv" + And Click plugin property: "skipHeader" + And Click on the Get Schema button + And Validate "File" plugin properties + And Close the Plugin Properties page + And Navigate to the properties page of plugin: "AddField" + And Enter input plugin property: "addFieldFieldName" with value: "afFieldName" + And Enter input plugin property: "addFieldFieldValue" with value: "afFieldValue" + And Validate "AddField" plugin properties + And Validate output schema with expectedSchema "csvAllDataTypeFileSchemaAddField" + And Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Replace input plugin property: "projectId" with value: "projectId" + And Enter input plugin property: "datasetProjectId" with value: "projectId" + Then Override Service account details if set in environment variables + And Enter input plugin property: "referenceName" with value: "BQReferenceName" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqTargetTable" + And Click plugin property: "truncateTable" + And Click plugin property: "updateTableSchema" + And Validate "BigQuery" plugin properties + And Close the Plugin Properties page + And Save the pipeline + And Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Wait till pipeline is in running state + And Open and capture logs + And Verify the pipeline status is "Succeeded" + Then Validate OUT record count is equal to IN record count + Then Verify column: "afFieldName" is added in target BigQuery table: "bqTargetTable" + + @BQ_SINK_TEST @FILE_SOURCE_TEST + Scenario: Verify add field plugin functionality by setting generate uuid as value to true using File to BigQuery pipeline + Given Open Datafusion Project to configure pipeline + And Select plugin: "File" from the plugins list as: "Source" + And Expand Plugin group in the LHS plugins list: "Transform" + And Select plugin: "Add Field" from the plugins list as: "Transform" + And Expand Plugin group in the LHS plugins list: "Sink" + And Select plugin: "BigQuery" from the plugins list as: "Sink" + And Connect plugins: "File" and "AddField" to establish connection + And Connect plugins: "AddField" and "BigQuery" to establish connection + And Navigate to the properties page of plugin: "File" + And Enter input plugin property: "referenceName" with value: "FileReferenceName" + And Enter input plugin property: "path" with value: "csvAllDataTypeFile" + And Select dropdown plugin property: "format" with option value: "csv" + And Click plugin property: "skipHeader" + And Click on the Get Schema button + And Validate "File" plugin properties + And Close the Plugin Properties page + And Navigate to the properties page of plugin: "AddField" + And Enter input plugin property: "addFieldFieldName" with value: "afFieldName" + And Select dropdown plugin property: "addFieldGenerateUUID" with option value: "true" + And Validate "AddField" plugin properties + And Validate output schema with expectedSchema "csvAllDataTypeFileSchemaAddField" + And Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Replace input plugin property: "projectId" with value: "projectId" + And Enter input plugin property: "datasetProjectId" with value: "projectId" + Then Override Service account details if set in environment variables + And Enter input plugin property: "referenceName" with value: "BQReferenceName" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqTargetTable" + And Click plugin property: "truncateTable" + And Click plugin property: "updateTableSchema" + And Validate "BigQuery" plugin properties + And Close the Plugin Properties page + And Save the pipeline + And Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Wait till pipeline is in running state + And Open and capture logs + And Verify the pipeline status is "Succeeded" + Then Validate OUT record count is equal to IN record count + Then Verify column: "afFieldName" is added in target BigQuery table: "bqTargetTable" diff --git a/src/e2e-test/java/io/cdap/plugin/addfield/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/addfield/runners/TestRunner.java new file mode 100644 index 0000000..d8332cd --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/addfield/runners/TestRunner.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.addfield.runners; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + +/** + * Test Runner to execute add-field plugin testcases. + */ +@RunWith(Cucumber.class) +@CucumberOptions( + features = {"src/e2e-test/features"}, + glue = {"io.cdap.plugin.addfield.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"}, + tags = {"@Add_Field"}, + monochrome = true, + plugin = {"pretty", "html:target/cucumber-html-report/add-field", + "json:target/cucumber-reports/cucumber-add-field.json", + "junit:target/cucumber-reports/cucumber-add-field.xml"} +) +public class TestRunner { +} diff --git a/src/e2e-test/java/io/cdap/plugin/addfield/runners/package-info.java b/src/e2e-test/java/io/cdap/plugin/addfield/runners/package-info.java new file mode 100644 index 0000000..579a8c5 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/addfield/runners/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** + * Package contains the runner for add-field plugin. + */ +package io.cdap.plugin.addfield.runners; diff --git a/src/e2e-test/java/io/cdap/plugin/addfield/stepsdesign/AddField.java b/src/e2e-test/java/io/cdap/plugin/addfield/stepsdesign/AddField.java new file mode 100644 index 0000000..56c5615 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/addfield/stepsdesign/AddField.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.addfield.stepsdesign; + +import io.cdap.e2e.utils.BigQueryClient; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cucumber.java.en.Then; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; + +/** + * Add-Field plugin related step design. + */ + +public class AddField { + + @Then("Verify column: {string} is added in target BigQuery table: {string}") + public void verifyColumnNameFromBigQueryTargetTable(String column, String tableName) throws IOException, + InterruptedException { + String columnName = PluginPropertyUtils.pluginProp(column); + Optional result = BigQueryClient + .getSoleQueryResult("SELECT column_name FROM `" + (PluginPropertyUtils.pluginProp("projectId")) + "." + + (PluginPropertyUtils.pluginProp("dataset")) + ".INFORMATION_SCHEMA.COLUMNS` " + + "WHERE table_name = '" + PluginPropertyUtils.pluginProp(tableName) + + "' and column_name = '" + columnName + "' "); + String targetTableColumnName = StringUtils.EMPTY; + if (result.isPresent()) { + targetTableColumnName = result.get(); + } + Assert.assertTrue("Column '" + columnName + "' should present in target BigQuery table", + targetTableColumnName.equalsIgnoreCase(columnName)); + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/addfield/stepsdesign/package-info.java b/src/e2e-test/java/io/cdap/plugin/addfield/stepsdesign/package-info.java new file mode 100644 index 0000000..5bd5071 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/addfield/stepsdesign/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** + * Package contains the stepDesign for the add-field plugin features. + */ +package io.cdap.plugin.addfield.stepsdesign; diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java new file mode 100644 index 0000000..eb45093 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.common.stepsdesign; + +import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.utils.BigQueryClient; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cucumber.java.After; +import io.cucumber.java.Before; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import stepsdesign.BeforeActions; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.UUID; + +/** + * GCP test hooks. + */ +public class TestSetupHooks { + private static boolean firstFileSourceTestFlag = true; + + @Before(order = 1, value = "@FILE_SOURCE_TEST") + public static void setFileSourceAbsolutePath() { + if (firstFileSourceTestFlag) { + PluginPropertyUtils.addPluginProp("csvAllDataTypeFile", Paths.get(TestSetupHooks.class.getResource + ("/" + PluginPropertyUtils.pluginProp("csvAllDataTypeFile")).getPath()).toString()); + firstFileSourceTestFlag = false; + } + } + + @Before(order = 1, value = "@BQ_SINK_TEST") + public static void setTempTargetBQTableName() { + String bqTargetTableName = "E2E_TARGET_" + UUID.randomUUID().toString().replaceAll("-", "_"); + PluginPropertyUtils.addPluginProp("bqTargetTable", bqTargetTableName); + BeforeActions.scenario.write("BQ Target table name - " + bqTargetTableName); + } + + @After(order = 1, value = "@BQ_SINK_TEST") + public static void deleteTempTargetBQTable() throws IOException, InterruptedException { + String bqTargetTableName = PluginPropertyUtils.pluginProp("bqTargetTable"); + try { + BigQueryClient.dropBqQuery(bqTargetTableName); + BeforeActions.scenario.write("BQ Target table - " + bqTargetTableName + " deleted successfully"); + PluginPropertyUtils.removePluginProp("bqTargetTable"); + } catch (BigQueryException e) { + if (e.getMessage().contains("Not found: Table")) { + BeforeActions.scenario.write("BQ Target Table " + bqTargetTableName + " does not exist"); + } else { + Assert.fail(e.getMessage()); + } + } + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java new file mode 100644 index 0000000..341ff96 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** + * Package contains the stepDesign for the common features. + */ +package io.cdap.plugin.common.stepsdesign; diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties new file mode 100644 index 0000000..573d54f --- /dev/null +++ b/src/e2e-test/resources/errorMessage.properties @@ -0,0 +1,5 @@ +validationSuccessMessage=No errors found. +validationErrorMessage=COUNT ERROR found + +addFieldErrorMessageInvalidField=Must specify a field value or set 'Generate UUID as Value' to true. +addFieldErrorMessageValidFieldValue=Must not specify a field value or set 'Generate UUID as Value' to false. diff --git a/src/e2e-test/resources/pluginDataCyAttributes.properties b/src/e2e-test/resources/pluginDataCyAttributes.properties new file mode 100644 index 0000000..4263353 --- /dev/null +++ b/src/e2e-test/resources/pluginDataCyAttributes.properties @@ -0,0 +1,16 @@ +referenceName=referenceName +path=path +format=select-format +skipHeader=switch-skipHeader +projectId=project +datasetProjectId=datasetProject +dataset=dataset +table=table +truncateTable=switch-truncateTable +updateTableSchema=switch-allowSchemaRelaxation + +## ADD_FIELD-DATA-CY-PROPERTIES-START +addFieldFieldName=fieldName +addFieldFieldValue=fieldValue +addFieldGenerateUUID=asUUID +## ADD_FIELD-DATA-CY-PROPERTIES-END diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties new file mode 100644 index 0000000..0beae28 --- /dev/null +++ b/src/e2e-test/resources/pluginParameters.properties @@ -0,0 +1,18 @@ +projectId=cdf-athena +dataset=test_automation +csvFormat=csv + +## FILE-PLUGIN-PROPERTIES-START +csvAllDataTypeFile=testdata/file/CSV_DATATYPE_TEST_1.csv +## FILE-PLUGIN-PROPERTIES-END + +## ADD-FIELD-PLUGIN-PROPERTIES-START +afFieldName=services +afFieldValue=nextforce +csvAllDataTypeFileSchemaAddField=[{"key":"id","value":"int"},{"key":"name","value":"string"},\ + {"key":"yearofbirth","value":"int"},{"key":"isdeleted","value":"boolean"},{"key":"email","value":"string"},\ + {"key":"createddate","value":"string"},{"key":"revenue","value":"string"},{"key":"points","value":"string"},\ + {"key":"longdatatype","value":"string"},{"key":"doubledatatype","value":"double"},\ + {"key":"date","value":"string"},{"key":"null","value":"string"},{"key":"BytesData","value":"string"},\ + {"key":"services","value":"string"}] +## ADD-FIELD-PLUGIN-PROPERTIES-END diff --git a/src/e2e-test/resources/testdata/file/CSV_DATATYPE_TEST_1.csv b/src/e2e-test/resources/testdata/file/CSV_DATATYPE_TEST_1.csv new file mode 100644 index 0000000..94b35af --- /dev/null +++ b/src/e2e-test/resources/testdata/file/CSV_DATATYPE_TEST_1.csv @@ -0,0 +1,5 @@ +id,name,yearofbirth,isdeleted,email,createddate,revenue,points,longdatatype,doubledatatype,date,null,BytesData +1,albert einstein,1879,true,sumitsri@gmail.com,2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-9223372036854770000,22.8,1996-07-21,,10111011101110111011 +2,isaac newton,1643,false,sumitsri@gmail.com,2021-09-20 11:27:50 UTC,900750000.01,3.14235678906787648,-9223372036854770000,123.08,1996-07-21,str,10111011101110111011 +3,marie curie,1867,true,sumitsri@gmail.com,2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-9223372036854770000,124.97,1996-07-21,,10111011101110111011 +4,galilée,1564,false,sumitsri@gmail.com,2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-2^63,234.89,1996-07-21,,10111011101110111011 \ No newline at end of file