From 601910a35c8fa9a469d1fa5067ead11ca45deead Mon Sep 17 00:00:00 2001 From: poojabasker20 Date: Fri, 12 May 2023 00:40:22 +0530 Subject: [PATCH 1/2] fix: updated validateInput method in CassandaToGCS --- .../templates/databases/CassandraToGCS.java | 5 ++- .../databases/CassandraToGCSTest.java | 36 +++++++++++++------ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java b/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java index fcc3bfcba..634ba876f 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java +++ b/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java @@ -78,5 +78,8 @@ public void runTemplate() throws StreamingQueryException, TimeoutException { .save(config.getOutputpath()); } - public void validateInput() {} + @Override + public void validateInput() { + ValidationUtil.validateOrThrow(config); + } } diff --git a/java/src/test/java/com/google/cloud/dataproc/templates/databases/CassandraToGCSTest.java b/java/src/test/java/com/google/cloud/dataproc/templates/databases/CassandraToGCSTest.java index f96f3c2e2..a3b5dc1a0 100644 --- a/java/src/test/java/com/google/cloud/dataproc/templates/databases/CassandraToGCSTest.java +++ b/java/src/test/java/com/google/cloud/dataproc/templates/databases/CassandraToGCSTest.java @@ -15,15 +15,21 @@ */ package com.google.cloud.dataproc.templates.databases; -import static com.google.cloud.dataproc.templates.util.TemplateConstants.*; -import static org.junit.jupiter.api.Assertions.*; +import static com.google.cloud.dataproc.templates.util.TemplateConstants.CASSANDRA_TO_GSC_INPUT_HOST; +import static com.google.cloud.dataproc.templates.util.TemplateConstants.CASSANDRA_TO_GSC_INPUT_KEYSPACE; +import static com.google.cloud.dataproc.templates.util.TemplateConstants.CASSANDRA_TO_GSC_INPUT_TABLE; +import static com.google.cloud.dataproc.templates.util.TemplateConstants.CASSANDRA_TO_GSC_OUTPUT_FORMAT; +import static com.google.cloud.dataproc.templates.util.TemplateConstants.CASSANDRA_TO_GSC_OUTPUT_PATH; +import static com.google.cloud.dataproc.templates.util.TemplateConstants.CASSANDRA_TO_GSC_OUTPUT_SAVE_MODE; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.cloud.dataproc.templates.util.PropertyUtil; import com.google.cloud.dataproc.templates.util.ValidationUtil.ValidationException; +import java.util.Properties; import java.util.stream.Stream; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.function.ThrowingSupplier; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; @@ -31,16 +37,12 @@ public class CassandraToGCSTest { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraToGCSTest.class); + final Properties properties = PropertyUtil.getProperties(); @BeforeEach void setUp() { SparkSession spark = SparkSession.builder().master("local").getOrCreate(); - } - @ParameterizedTest - @MethodSource("propertyKeys") - void runTemplateWithValidParameters(String propKey) { - LOGGER.info("Running test: runTemplateWithValidParameters"); PropertyUtil.getProperties() .setProperty(CASSANDRA_TO_GSC_OUTPUT_PATH, "gs://test-bucket/output"); PropertyUtil.getProperties().setProperty(CASSANDRA_TO_GSC_OUTPUT_SAVE_MODE, "Append"); @@ -48,15 +50,27 @@ void runTemplateWithValidParameters(String propKey) { PropertyUtil.getProperties().setProperty(CASSANDRA_TO_GSC_INPUT_HOST, "host"); PropertyUtil.getProperties().setProperty(CASSANDRA_TO_GSC_INPUT_TABLE, "table"); PropertyUtil.getProperties().setProperty(CASSANDRA_TO_GSC_INPUT_KEYSPACE, "keyspace"); - assertDoesNotThrow((ThrowingSupplier) CassandraToGCS::of); + } + + @ParameterizedTest + @MethodSource("propertyKeys") + void runTemplateWithValidParameters(String propKey) { + LOGGER.info("Running test: runTemplateWithValidParameters"); + + CassandraToGCSConfig config = CassandraToGCSConfig.fromProperties(PropertyUtil.getProperties()); + CassandraToGCS template = new CassandraToGCS(config); + assertDoesNotThrow(template::validateInput); } @ParameterizedTest @MethodSource("propertyKeys") void runTemplateWithInvalidParameters(String propKey) { LOGGER.info("Running test: runTemplateWithInvalidParameters"); - PropertyUtil.getProperties().setProperty(propKey, ""); - ValidationException exception = assertThrows(ValidationException.class, CassandraToGCS::of); + properties.setProperty(propKey, ""); + CassandraToGCSConfig config = CassandraToGCSConfig.fromProperties(PropertyUtil.getProperties()); + CassandraToGCS template = new CassandraToGCS(config); + ValidationException exception = + assertThrows(ValidationException.class, template::validateInput); } static Stream propertyKeys() { From beb3d9d9a3a74cfcfbf096412cfe15baf33da594 Mon Sep 17 00:00:00 2001 From: poojabasker20 Date: Fri, 12 May 2023 11:36:08 +0530 Subject: [PATCH 2/2] fix: remove validateOrThrow call from of() --- .../cloud/dataproc/templates/databases/CassandraToGCS.java | 1 - 1 file changed, 1 deletion(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java b/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java index 634ba876f..6946ba248 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java +++ b/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToGCS.java @@ -38,7 +38,6 @@ public CassandraToGCS(CassandraToGCSConfig config) { public static CassandraToGCS of(String... args) { CassandraToGCSConfig config = CassandraToGCSConfig.fromProperties(PropertyUtil.getProperties()); - ValidationUtil.validateOrThrow(config); LOGGER.info("Config loaded\n{}", config); return new CassandraToGCS(config); }