Merge branch 'main' into 600-java-template-gcstospanner-empty-validat…

…einput-method
GoogleCloudPlatform · May 11, 2023 · ab5a7e8 · ab5a7e8
2 parents 19a052b + c67ae7d
commit ab5a7e8
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -54,7 +54,7 @@ Please refer to the [Dataproc Templates (Python - PySpark) README](/python) for
 * [AzureBlobToBigQuery](/python/dataproc_templates/azure#azure-blob-to-bigquery)
 * [BigQueryToGCS](/python/dataproc_templates/bigquery#bigquery-to-gcs) (blogpost [link](https://medium.com/google-cloud/moving-data-from-bigquery-to-gcs-using-gcp-dataproc-serverless-and-pyspark-f6481b86bcd1))
 * [CassandraToBigquery](/python/dataproc_templates/cassandra)
-* [CassandraToGCS](/python/dataproc_templates/cassandra)
+* [CassandraToGCS](/python/dataproc_templates/cassandra) (blogpost [link](https://medium.com/google-cloud/export-data-from-cassandra-to-google-cloud-storage-using-dataproc-serverless-2569a00e17fe))
 * [GCSToBigQuery](/python/dataproc_templates/gcs#gcs-to-bigquery) (blogpost [link](https://medium.com/@ppaglilla/getting-started-with-dataproc-serverless-pyspark-templates-e32278a6a06e))
 * [GCSToBigTable](/python/dataproc_templates/gcs#gcs-to-bigtable)(blogpost [link](https://medium.com/google-cloud/pyspark-load-data-from-gcs-to-bigtable-using-gcp-dataproc-serverless-c373430fe157))
 * [GCSToGCS](/python/dataproc_templates/gcs#gcs-to-gcs---sql-transformation) (blogpost [link](https://medium.com/@ankuljain/migrate-gcs-to-gcs-using-dataproc-serverless-3b7b0f6ad6b9))

diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToBQ.java b/java/src/main/java/com/google/cloud/dataproc/templates/databases/CassandraToBQ.java
@@ -15,6 +15,8 @@
  */
 package com.google.cloud.dataproc.templates.databases;
 
+import static com.google.cloud.dataproc.templates.util.TemplateConstants.*;
+
 import com.datastax.spark.connector.CassandraSparkExtensions;
 import com.google.cloud.dataproc.templates.BaseTemplate;
 import com.google.cloud.dataproc.templates.util.PropertyUtil;
@@ -37,7 +39,6 @@ public CassandraToBQ(CassandraToBqConfig config) {
 
   public static CassandraToBQ of(String... args) {
     CassandraToBqConfig config = CassandraToBqConfig.fromProperties(PropertyUtil.getProperties());
-    ValidationUtil.validateOrThrow(config);
     LOGGER.info("Config loaded\n{}", config);
     return new CassandraToBQ(config);
   }
@@ -80,5 +81,8 @@ public void runTemplate() throws StreamingQueryException, TimeoutException {
         .save();
   }
 
-  public void validateInput() {}
+  @Override
+  public void validateInput() {
+    ValidationUtil.validateOrThrow(config);
+  }
 }
diff --git a/java/src/test/java/com/google/cloud/dataproc/templates/databases/CassandraToBQTest.java b/java/src/test/java/com/google/cloud/dataproc/templates/databases/CassandraToBQTest.java
@@ -23,7 +23,6 @@
 import java.util.stream.Stream;
 import org.apache.spark.sql.SparkSession;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.function.ThrowingSupplier;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.slf4j.Logger;
@@ -47,15 +46,20 @@ void runTemplateWithValidParameters(String propKey) {
     PropertyUtil.getProperties().setProperty(CASSANDRA_TO_BQ_INPUT_HOST, "host");
     PropertyUtil.getProperties().setProperty(CASSANDRA_TO_BQ_INPUT_TABLE, "table");
     PropertyUtil.getProperties().setProperty(CASSANDRA_TO_BQ_INPUT_KEYSPACE, "keyspace");
-    assertDoesNotThrow((ThrowingSupplier<CassandraToBQ>) CassandraToBQ::of);
+    CassandraToBqConfig config = CassandraToBqConfig.fromProperties(PropertyUtil.getProperties());
+    CassandraToBQ template = new CassandraToBQ(config);
+    assertDoesNotThrow(template::validateInput);
   }
 
   @ParameterizedTest
   @MethodSource("propertyKeys")
   void runTemplateWithInvalidParameters(String propKey) {
     LOGGER.info("Running test: runTemplateWithInvalidParameters");
     PropertyUtil.getProperties().setProperty(propKey, "");
-    ValidationException exception = assertThrows(ValidationException.class, CassandraToBQ::of);
+    CassandraToBqConfig config = CassandraToBqConfig.fromProperties(PropertyUtil.getProperties());
+    CassandraToBQ template = new CassandraToBQ(config);
+    ValidationException exception =
+        assertThrows(ValidationException.class, template::validateInput);
   }
 
   static Stream<String> propertyKeys() {

diff --git a/python/README.md b/python/README.md
@@ -6,7 +6,7 @@
 * [AzureBlobStorageToBigQuery](/python/dataproc_templates/azure#azure-blob-storage-to-bigquery)
 * [BigQueryToGCS](/python/dataproc_templates/bigquery#bigquery-to-gcs) (blogpost [link](https://medium.com/google-cloud/moving-data-from-bigquery-to-gcs-using-gcp-dataproc-serverless-and-pyspark-f6481b86bcd1))
 * [CassandraToBigquery](/python/dataproc_templates/cassandra#cassandra-to-bigquery)
-* [CassandraToGCS](/python/dataproc_templates/cassandra#cassandra-to-gcs)
+* [CassandraToGCS](/python/dataproc_templates/cassandra#cassandra-to-gcs) (blogpost [link](https://medium.com/google-cloud/export-data-from-cassandra-to-google-cloud-storage-using-dataproc-serverless-2569a00e17fe))
 * [GCSToBigQuery](/python/dataproc_templates/gcs#gcs-to-bigquery) (blogpost [link](https://medium.com/@ppaglilla/getting-started-with-dataproc-serverless-pyspark-templates-e32278a6a06e))
 * [GCSToBigTable](/python/dataproc_templates/gcs#gcs-to-bigtable) (blogpost [link](https://medium.com/google-cloud/pyspark-load-data-from-gcs-to-bigtable-using-gcp-dataproc-serverless-c373430fe157))
 * [GCSToGCS](/python/dataproc_templates/gcs#gcs-to-gcs---sql-transformation)(blogpost [link](https://medium.com/@ankuljain/migrate-gcs-to-gcs-using-dataproc-serverless-3b7b0f6ad6b9))