From 6cf1275e59b980fcc0e0a91b3c8ddd69cb9910d2 Mon Sep 17 00:00:00 2001 From: saumyasinha-google <118509792+saumyasinha-google@users.noreply.github.com> Date: Tue, 28 Feb 2023 14:47:33 +0530 Subject: [PATCH 1/4] fix: Added validateInput method --- .../cloud/dataproc/templates/databases/RedshiftToGCS.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java b/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java index 12d2fa18d..0ced87815 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java +++ b/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java @@ -35,7 +35,6 @@ public RedshiftToGCS(RedshiftToGCSConfig config) { public static com.google.cloud.dataproc.templates.databases.RedshiftToGCS of(String... args) { RedshiftToGCSConfig config = RedshiftToGCSConfig.fromProperties(PropertyUtil.getProperties()); - ValidationUtil.validateOrThrow(config); LOGGER.info("Config loaded\n{}", config); return new com.google.cloud.dataproc.templates.databases.RedshiftToGCS(config); } @@ -82,5 +81,7 @@ public void runTemplate() { spark.stop(); } - public void validateInput() {} + public void validateInput() { + ValidationUtil.validateOrThrow(config); + } } From ed44c2770bb4597f259adcb87e5840a8131ca146 Mon Sep 17 00:00:00 2001 From: saumyasinha-google Date: Thu, 9 Mar 2023 12:28:36 +0530 Subject: [PATCH 2/4] fix: fixed redshiftToGCS template --- .../cloud/dataproc/templates/databases/RedshiftToGCS.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java b/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java index 0ced87815..2db1046c3 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java +++ b/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java @@ -35,13 +35,14 @@ public RedshiftToGCS(RedshiftToGCSConfig config) { public static com.google.cloud.dataproc.templates.databases.RedshiftToGCS of(String... args) { RedshiftToGCSConfig config = RedshiftToGCSConfig.fromProperties(PropertyUtil.getProperties()); + RedshiftToGCS redshiftToGCS = new RedshiftToGCS(config); + redshiftToGCS.validateInput(); LOGGER.info("Config loaded\n{}", config); - return new com.google.cloud.dataproc.templates.databases.RedshiftToGCS(config); + return redshiftToGCS; } @Override public void runTemplate() { - validateInput(); SparkSession spark = SparkSession.builder() .appName("Spark Template RedshiftToGCS ") From 49b55d604259428fc9b0aa402db7e54f12655625 Mon Sep 17 00:00:00 2001 From: Vanshaj Bhatia Date: Thu, 11 May 2023 15:28:09 +0000 Subject: [PATCH 3/4] fix: improved usage of validateInput --- .../cloud/dataproc/templates/databases/RedshiftToGCS.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java b/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java index 779f2a8bb..a4d6155dd 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java +++ b/java/src/main/java/com/google/cloud/dataproc/templates/databases/RedshiftToGCS.java @@ -35,10 +35,8 @@ public RedshiftToGCS(RedshiftToGCSConfig config) { public static com.google.cloud.dataproc.templates.databases.RedshiftToGCS of(String... args) { RedshiftToGCSConfig config = RedshiftToGCSConfig.fromProperties(PropertyUtil.getProperties()); - RedshiftToGCS redshiftToGCS = new RedshiftToGCS(config); - redshiftToGCS.validateInput(); LOGGER.info("Config loaded\n{}", config); - return redshiftToGCS; + return new RedshiftToGCS(config); } @Override @@ -83,6 +81,7 @@ public void runTemplate() { spark.stop(); } + @Override public void validateInput() { ValidationUtil.validateOrThrow(config); } From 038e5e824fb9b8150897d9f428a86c70b78e1199 Mon Sep 17 00:00:00 2001 From: Vanshaj Bhatia Date: Thu, 11 May 2023 15:29:27 +0000 Subject: [PATCH 4/4] fix: updated unit test cases for RedshiftToGCS --- .../templates/redshift/RedshiftToGCSTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/java/src/test/java/com/google/cloud/dataproc/templates/redshift/RedshiftToGCSTest.java b/java/src/test/java/com/google/cloud/dataproc/templates/redshift/RedshiftToGCSTest.java index 1cb78691c..c6fb8b18a 100644 --- a/java/src/test/java/com/google/cloud/dataproc/templates/redshift/RedshiftToGCSTest.java +++ b/java/src/test/java/com/google/cloud/dataproc/templates/redshift/RedshiftToGCSTest.java @@ -21,12 +21,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.cloud.dataproc.templates.databases.RedshiftToGCS; +import com.google.cloud.dataproc.templates.databases.RedshiftToGCSConfig; import com.google.cloud.dataproc.templates.util.PropertyUtil; import com.google.cloud.dataproc.templates.util.ValidationUtil.ValidationException; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.function.ThrowingSupplier; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; @@ -54,7 +54,9 @@ void setUp() { @Test void runTemplateWithValidParameters() { LOGGER.info("Running test: runTemplateWithValidParameters"); - assertDoesNotThrow((ThrowingSupplier) RedshiftToGCS::of); + RedshiftToGCSConfig config = RedshiftToGCSConfig.fromProperties(PropertyUtil.getProperties()); + RedshiftToGCS template = new RedshiftToGCS(config); + assertDoesNotThrow(template::validateInput); } @ParameterizedTest @@ -62,7 +64,10 @@ void runTemplateWithValidParameters() { void runTemplateWithMissingRequiredParameters(String propKey) { LOGGER.info("Running test: runTemplateWithInvalidParameters"); PropertyUtil.getProperties().setProperty(propKey, ""); - ValidationException exception = assertThrows(ValidationException.class, RedshiftToGCS::of); + RedshiftToGCSConfig config = RedshiftToGCSConfig.fromProperties(PropertyUtil.getProperties()); + RedshiftToGCS template = new RedshiftToGCS(config); + ValidationException exception = + assertThrows(ValidationException.class, template::validateInput); } static Stream requiredPropertyKeys() {