Update to 0.11.5, change ODCS file to use .odcs.yaml suffix, update p…

…lans with correct foreign key definition
data-catering · Jul 3, 2024 · a9d158c · a9d158c
1 parent c481d12
commit a9d158c
Show file tree

Hide file tree

Showing 10 changed files with 11 additions and 5 deletions.
diff --git a/docker/data/custom/plan/cassandra.yaml b/docker/data/custom/plan/cassandra.yaml
@@ -7,5 +7,6 @@ tasks:
 
 sinkOptions:
   foreignKeys:
-    "cassandra.accounts.account_id":
-      - "cassandra.account_status_history.account_id"
+    - - "cassandra.accounts.account_id"
+      - - "cassandra.account_status_history.account_id"
+      - []
diff --git a/docker/data/custom/plan/foreign-key-example.yaml b/docker/data/custom/plan/foreign-key-example.yaml
@@ -12,3 +12,4 @@ sinkOptions:
   foreignKeys:
     - - "postgresCustomer.accounts.account_number"
       - - "json.account.account_id"
+      - []
diff --git a/docker/data/custom/plan/foreign-key.yaml b/docker/data/custom/plan/foreign-key.yaml
@@ -12,3 +12,4 @@ sinkOptions:
   foreignKeys:
     - - "postgresCustomer.accounts.account_number"
       - - "json.account.account_id"
+      - []
diff --git a/docker/data/custom/plan/postgres-multiple-tables.yaml b/docker/data/custom/plan/postgres-multiple-tables.yaml
@@ -9,3 +9,4 @@ sinkOptions:
   foreignKeys:
     - - "postgresCustomer.balances.account_number"
       - - "postgresCustomer.transactions.account_number"
+      - []
diff --git a/docker/data/custom/plan/scenario-based-parquet.yaml b/docker/data/custom/plan/scenario-based-parquet.yaml
@@ -12,3 +12,4 @@ sinkOptions:
   foreignKeys:
     - - "postgres.accounts.account_number"
       - - "parquet.transactions.account_id"
+      - []
diff --git a/docker/data/custom/plan/scenario-based.yaml b/docker/data/custom/plan/scenario-based.yaml
@@ -12,3 +12,4 @@ sinkOptions:
   foreignKeys:
     - - "postgresCustomer.accounts.account_number"
       - - "csv.transactions.account_id"
+      - []
diff --git a/docker/mount/odcs/full-example.yaml → docker/mount/odcs/full-example.odcs.yaml b/docker/mount/odcs/full-example.yaml → docker/mount/odcs/full-example.odcs.yaml
diff --git a/gradle.properties b/gradle.properties
@@ -8,6 +8,6 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.15
-dataCatererVersion=0.11.2
+dataCatererVersion=0.11.5
 sparkVersion=3.5.1
 sparkMajorVersion=3.5
diff --git a/src/main/java/io/github/datacatering/plan/AdvancedODCSJavaPlanRun.java b/src/main/java/io/github/datacatering/plan/AdvancedODCSJavaPlanRun.java
@@ -7,7 +7,7 @@
 public class AdvancedODCSJavaPlanRun extends PlanRun {
     {
         var accountTask = csv("my_csv", "/opt/app/data/account-odcs", Map.of("saveMode", "overwrite", "header", "true"))
-                .schema(metadataSource().openDataContractStandard("/opt/app/mount/odcs/full-example.yaml"))
+                .schema(metadataSource().openDataContractStandard("/opt/app/mount/odcs/full-example.odcs.yaml"))
                 .count(count().records(100));
 
         var conf = configuration().enableGeneratePlanAndTasks(true)

diff --git a/src/main/scala/io/github/datacatering/plan/AdvancedODCSPlanRun.scala b/src/main/scala/io/github/datacatering/plan/AdvancedODCSPlanRun.scala
@@ -5,7 +5,7 @@ import io.github.datacatering.datacaterer.api.PlanRun
 class AdvancedODCSPlanRun extends PlanRun {
 
   val accountTask = csv("customer_accounts", "/opt/app/data/customer/account-odcs", Map("header" -> "true"))
-    .schema(metadataSource.openDataContractStandard("/opt/app/mount/odcs/full-example.yaml"))
+    .schema(metadataSource.openDataContractStandard("/opt/app/mount/odcs/full-example.odcs.yaml"))
     .count(count.records(100))
 
   val conf = configuration.enableGeneratePlanAndTasks(true)