Pub/Sub Example (#63)

.gitignore

@@ -10,3 +10,10 @@ config/.terraform*
 
 .personal/
 scratch/
+
+# Local .terraform directories
+**/.terraform.lock.hcl
+**/.terraform/*
+# .tfstate files
+*.tfstate
+*.tfstate.*

examples/pubsub_postgres/Dockerfile

@@ -0,0 +1,5 @@
+FROM artielabs/transfer:latest
+
+COPY . .
+
+CMD ["/transfer", "--config", "config.yaml"]

examples/pubsub_postgres/README.md

@@ -0,0 +1,3 @@
+# Postgres Example
+
+This example requires additional configuration on the Pub/Sub side. Please see https://docs.artie.so/configurations/wip-tutorials/setting-up-pub-sub for further details.

examples/pubsub_postgres/application.properties

@@ -0,0 +1,19 @@
+# Offset storage
+debezium.source.offset.storage.file.filename=/tmp/foo
+debezium.source.offset.flush.interval.ms=0
+
+# Pubsub setup: https://debezium.io/documentation/reference/stable/operations/debezium-server.html#_google_cloud_pubsub
+debezium.sink.type=pubsub
+debezium.sink.pubsub.project.id=PROJECT_ID
+debezium.sink.pubsub.ordering.enabled=true
+
+# Postgres
+debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
+debezium.source.database.hostname=postgres
+debezium.source.database.port=5432
+debezium.source.database.user=postgres
+debezium.source.database.password=postgres
+debezium.source.database.dbname=postgres
+debezium.source.topic.prefix=dbserver1
+debezium.source.table.include.list=inventory.customers
+debezium.source.plugin.name=pgoutput

examples/pubsub_postgres/config.yaml

@@ -0,0 +1,22 @@
+outputSource: test
+queue: pubsub
+
+pubsub:
+  projectID: artie-labs
+  pathToCredentials: /tmp/credentials/service-account.json
+  topicConfigs:
+    - db: customers
+      tableName: customers
+      schema: public
+      topic: "dbserver1.inventory.customers"
+      cdcFormat: debezium.postgres.wal2json
+      cdcKeyFormat: org.apache.kafka.connect.json.JsonConverter
+
+telemetry:
+  metrics:
+    provider: datadog
+    settings:
+      tags:
+       - env:production
+      namespace: "transfer."
+      addr: "127.0.0.1:8125"

examples/pubsub_postgres/docker-compose.yaml

@@ -0,0 +1,31 @@
+version: '3.9'
+services:
+  postgres:
+    image: quay.io/debezium/example-postgres:2.0
+    ports:
+     - 5432:5432
+    environment:
+     - POSTGRES_USER=postgres
+     - POSTGRES_PASSWORD=postgres
+  debezium-server:
+    image: quay.io/debezium/server:2.0
+    container_name: debezium-server
+    # Sleep the PostgreSQL service to spin up.
+    command: sh -c "sleep 15 && /debezium/run.sh"
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /tmp/credentials/service-account.json
+    links:
+      - postgres
+    ports:
+      - 8080:8080
+    volumes:
+      - ./application.properties:/debezium/conf/application.properties
+      - REPLACE_ME:/tmp/credentials/service-account.json
+    depends_on:
+      - postgres
+  transfer:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - REPLACE_ME:/tmp/credentials/service-account.json

examples/pubsub_postgres/tf/main.tf

@@ -0,0 +1,49 @@
+# In here, we will be doing the following:
+# 1) Creating a service account
+# 2) Creating a topic
+# 3) Creating a subscription
+# Note: - We require a topic per table that we want to replicate from.
+locals {
+  project = "PROJECT_ID"
+  role = "roles/pubsub.editor"
+}
+
+provider "google" {
+  project     = local.project
+  // Authenticate via gcloud auth application-default login
+  // This requires the gcloud CLI downloaded: https://cloud.google.com/sdk/docs/install
+  // Need to enable PubSub API: https://console.cloud.google.com/marketplace/product/google/pubsub.googleapis.com
+}
+
+resource "google_service_account" "artie-svc-account" {
+  account_id   = "artie-service-account"
+  display_name = "Service Account for Artie Transfer and Debezium"
+}
+
+resource "google_project_iam_member" "transfer" {
+  project = local.project
+  role    = local.role
+  member  = "serviceAccount:${google_service_account.artie-svc-account.email}"
+}
+
+# Pub/Sub configurations
+# We will create the topic and subscription.
+resource "google_pubsub_topic" "customer_topic" {
+  name    = "dbserver1.inventory.customers"
+  project = local.project
+
+  timeouts {}
+}
+
+resource "google_pubsub_subscription" "customer_subscription" {
+  ack_deadline_seconds         = 300
+  enable_exactly_once_delivery = false
+  enable_message_ordering      = true
+  message_retention_duration   = "604800s"
+  name                         = "transfer_${google_pubsub_topic.customer_topic.name}"
+  project                      = local.project
+  retain_acked_messages        = false
+  topic                        = google_pubsub_topic.customer_topic.id
+
+  timeouts {}
+}