Merge pull request #24 from artie-labs/dana/db-config-schema

Create db-specific config schemas
artie-labs · Aug 6, 2024 · 48c8e78 · 48c8e78
2 parents 2391686 + cacbcda
commit 48c8e78
Show file tree

Hide file tree

Showing 11 changed files with 348 additions and 286 deletions.
diff --git a/docs/resources/deployment.md b/docs/resources/deployment.md
@@ -51,38 +51,13 @@ Optional:
 
 Required:
 
-- `config` (Attributes) (see [below for nested schema](#nestedatt--source--config))
-- `name` (String)
 - `tables` (Attributes List) (see [below for nested schema](#nestedatt--source--tables))
-
-<a id="nestedatt--source--config"></a>
-### Nested Schema for `source.config`
-
-Required:
-
-- `database` (String)
-- `host` (String)
-- `password` (String, Sensitive)
-- `port` (Number)
-- `user` (String)
-
-Optional:
-
-- `dynamodb` (Attributes) (see [below for nested schema](#nestedatt--source--config--dynamodb))
-- `snapshot_host` (String)
-
-<a id="nestedatt--source--config--dynamodb"></a>
-### Nested Schema for `source.config.dynamodb`
+- `type` (String)
 
 Optional:
 
-- `aws_access_key_id` (String)
-- `aws_secret_access_key` (String)
-- `region` (String)
-- `streams_arn` (String)
-- `table_name` (String)
-
-
+- `mysql_config` (Attributes) (see [below for nested schema](#nestedatt--source--mysql_config))
+- `postgres_config` (Attributes) (see [below for nested schema](#nestedatt--source--postgres_config))
 
 <a id="nestedatt--source--tables"></a>
 ### Nested Schema for `source.tables`
@@ -101,3 +76,27 @@ Optional:
 Read-Only:
 
 - `uuid` (String)
+
+
+<a id="nestedatt--source--mysql_config"></a>
+### Nested Schema for `source.mysql_config`
+
+Required:
+
+- `database` (String)
+- `host` (String)
+- `password` (String, Sensitive)
+- `port` (Number)
+- `user` (String)
+
+
+<a id="nestedatt--source--postgres_config"></a>
+### Nested Schema for `source.postgres_config`
+
+Required:
+
+- `database` (String)
+- `host` (String)
+- `password` (String, Sensitive)
+- `port` (Number)
+- `user` (String)
diff --git a/docs/resources/destination.md b/docs/resources/destination.md
@@ -17,35 +17,53 @@ Artie Destination resource
 
 ### Required
 
-- `config` (Attributes) (see [below for nested schema](#nestedatt--config))
-- `name` (String)
+- `type` (String)
 
 ### Optional
 
+- `big_query_config` (Attributes) (see [below for nested schema](#nestedatt--big_query_config))
 - `label` (String)
+- `redshift_config` (Attributes) (see [below for nested schema](#nestedatt--redshift_config))
+- `snowflake_config` (Attributes) (see [below for nested schema](#nestedatt--snowflake_config))
 
 ### Read-Only
 
 - `company_uuid` (String)
 - `ssh_tunnel_uuid` (String)
 - `uuid` (String)
 
-<a id="nestedatt--config"></a>
-### Nested Schema for `config`
+<a id="nestedatt--big_query_config"></a>
+### Nested Schema for `big_query_config`
 
-Optional:
+Required:
+
+- `credentials_data` (String, Sensitive)
+- `location` (String)
+- `project_id` (String)
+
+
+<a id="nestedatt--redshift_config"></a>
+### Nested Schema for `redshift_config`
+
+Required:
 
-- `aws_access_key_id` (String)
-- `aws_region` (String)
-- `aws_secret_access_key` (String, Sensitive)
 - `endpoint` (String)
-- `gcp_credentials_data` (String, Sensitive)
-- `gcp_location` (String)
-- `gcp_project_id` (String)
 - `host` (String)
 - `password` (String, Sensitive)
 - `port` (Number)
-- `snowflake_account_url` (String)
-- `snowflake_private_key` (String, Sensitive)
-- `snowflake_virtual_dwh` (String)
 - `username` (String)
+
+
+<a id="nestedatt--snowflake_config"></a>
+### Nested Schema for `snowflake_config`
+
+Required:
+
+- `account_url` (String)
+- `username` (String)
+- `virtual_dwh` (String)
+
+Optional:
+
+- `password` (String, Sensitive)
+- `private_key` (String, Sensitive)
diff --git a/examples/deployments/main.tf b/examples/deployments/main.tf
@@ -10,115 +10,115 @@ provider "artie" {
   endpoint = "http://0.0.0.0:8000"
 }
 
-# import {
-#   to = artie_destination.snowflake
-#   id = "51b180a0-fbb9-49a2-ab45-cb46d913416d"
-# }
+import {
+  to = artie_destination.snowflake
+  id = "51b180a0-fbb9-49a2-ab45-cb46d913416d"
+}
+
+import {
+  to = artie_deployment.dev_postgres_to_snowflake
+  id = "c3dfa503-b6ae-48f3-a6b1-8491a506126d"
+}
+
+variable "snowflake_password" {
+  type      = string
+  sensitive = true
+}
+variable "postgres_password" {
+  type      = string
+  sensitive = true
+}
+
+resource "artie_destination" "snowflake" {
+  type  = "Snowflake"
+  label = "Snowflake (Partner Account)"
+  snowflake_config = {
+    account_url = "https://znb46775.snowflakecomputing.com"
+    virtual_dwh = "compute_wh"
+    username    = "tang8330"
+    password    = var.snowflake_password
+  }
+}
+
+resource "artie_deployment" "dev_postgres_to_snowflake" {
+  name = "Dev PostgreSQL > Snowflake"
+  source = {
+    type = "PostgreSQL"
+    postgres_config = {
+      host     = "db-postgresql-sfo3-03243-do-user-13261354-0.c.db.ondigitalocean.com"
+      port     = 25060
+      database = "prod_dump_july_2024_4cvzb"
+      user     = "doadmin"
+      password = var.postgres_password
+    }
+    tables = [
+      {
+        name   = "invite"
+        schema = "public"
+      }
+    ]
+  }
+  destination_uuid = artie_destination.snowflake.uuid
+  destination_config = {
+    database = "DEV_TEST"
+    schema   = "PUBLIC"
+  }
+}
 
 # import {
-#   to = artie_deployment.dev_postgres_to_snowflake
-#   id = "c3dfa503-b6ae-48f3-a6b1-8491a506126d"
+#   to = artie_destination.bigquery
+#   id = "fa7d4efc-3957-41e5-b29c-66e2d49bffde"
 # }
 
-# variable "snowflake_password" {
+# variable "mongodb_password" {
 #   type      = string
 #   sensitive = true
 # }
-# variable "postgres_password" {
+
+# variable "gcp_creds" {
 #   type      = string
 #   sensitive = true
 # }
 
-# resource "artie_destination" "snowflake" {
-#   name  = "Snowflake"
-#   label = "Snowflake (Partner Account)"
+# resource "artie_destination" "bigquery" {
+#   name  = "BigQuery"
+#   label = "BigQuery"
 #   config = {
-#     snowflake_account_url = "https://znb46775.snowflakecomputing.com"
-#     username              = "tang8330"
-#     password              = var.snowflake_password
-#     snowflake_virtual_dwh = "compute_wh"
+#     gcp_location         = "us"
+#     gcp_project_id       = "artie-labs"
+#     gcp_credentials_data = var.gcp_creds
 #   }
 # }
 
-# resource "artie_deployment" "dev_postgres_to_snowflake" {
-#   name = "Dev PostgreSQL > Snowflake"
+# import {
+#   to = artie_deployment.example
+#   id = "38d5d2db-870a-4a38-a76c-9891b0e5122d"
+# }
+
+# resource "artie_deployment" "example" {
+#   name = "MongoDB ➡️ BigQuery"
 #   source = {
-#     name = "PostgreSQL"
+#     name = "MongoDB"
 #     config = {
-#       host     = "db-postgresql-sfo3-03243-do-user-13261354-0.c.db.ondigitalocean.com"
-#       port     = 25060
-#       database = "prod_dump_july_2024_4cvzb"
-#       user     = "doadmin"
-#       password = var.postgres_password
+#       database = "myFirstDatabase"
+#       host     = "mongodb+srv://cluster0.szddg49.mongodb.net/"
+#       port     = 0
+#       user     = "artie"
+#       password = var.mongodb_password
 #     }
 #     tables = [
 #       {
-#         name   = "invite"
-#         schema = "public"
+#         name   = "customers"
+#         schema = ""
+#       },
+#       {
+#         name   = "stock"
+#         schema = ""
 #       }
 #     ]
 #   }
-#   destination_uuid = artie_destination.snowflake.uuid
+#   destination_uuid = artie_destination.bigquery.uuid
 #   destination_config = {
-#     database = "DEV_TEST"
-#     schema   = "PUBLIC"
+#     dataset = "customers"
 #   }
 # }
-
-import {
-  to = artie_destination.bigquery
-  id = "fa7d4efc-3957-41e5-b29c-66e2d49bffde"
-}
-
-variable "mongodb_password" {
-  type      = string
-  sensitive = true
-}
-
-variable "gcp_creds" {
-  type      = string
-  sensitive = true
-}
-
-resource "artie_destination" "bigquery" {
-  name  = "BigQuery"
-  label = "BigQuery"
-  config = {
-    gcp_location         = "us"
-    gcp_project_id       = "artie-labs"
-    gcp_credentials_data = var.gcp_creds
-  }
-}
-
-import {
-  to = artie_deployment.example
-  id = "38d5d2db-870a-4a38-a76c-9891b0e5122d"
-}
-
-resource "artie_deployment" "example" {
-  name = "MongoDB ➡️ BigQuery"
-  source = {
-    name = "MongoDB"
-    config = {
-      database = "myFirstDatabase"
-      host     = "mongodb+srv://cluster0.szddg49.mongodb.net/"
-      port     = 0
-      user     = "artie"
-      password = var.mongodb_password
-    }
-    tables = [
-      {
-        name   = "customers"
-        schema = ""
-      },
-      {
-        name   = "stock"
-        schema = ""
-      }
-    ]
-  }
-  destination_uuid = artie_destination.bigquery.uuid
-  destination_config = {
-    dataset = "customers"
-  }
-}