From b8da1ada1a8bb5cfe8eba86ff0945af7958ae32c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 7 Mar 2023 16:24:16 -0800 Subject: [PATCH] Pub/Sub Example (#63) --- .gitignore | 7 +++ examples/pubsub_postgres/Dockerfile | 5 ++ examples/pubsub_postgres/README.md | 3 ++ .../pubsub_postgres/application.properties | 19 +++++++ examples/pubsub_postgres/config.yaml | 22 +++++++++ examples/pubsub_postgres/docker-compose.yaml | 31 ++++++++++++ examples/pubsub_postgres/tf/main.tf | 49 +++++++++++++++++++ 7 files changed, 136 insertions(+) create mode 100644 examples/pubsub_postgres/Dockerfile create mode 100644 examples/pubsub_postgres/README.md create mode 100644 examples/pubsub_postgres/application.properties create mode 100644 examples/pubsub_postgres/config.yaml create mode 100644 examples/pubsub_postgres/docker-compose.yaml create mode 100644 examples/pubsub_postgres/tf/main.tf diff --git a/.gitignore b/.gitignore index 45a441b49..898854109 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,10 @@ config/.terraform* .personal/ scratch/ + +# Local .terraform directories +**/.terraform.lock.hcl +**/.terraform/* +# .tfstate files +*.tfstate +*.tfstate.* diff --git a/examples/pubsub_postgres/Dockerfile b/examples/pubsub_postgres/Dockerfile new file mode 100644 index 000000000..f05fc173d --- /dev/null +++ b/examples/pubsub_postgres/Dockerfile @@ -0,0 +1,5 @@ +FROM artielabs/transfer:latest + +COPY . . + +CMD ["/transfer", "--config", "config.yaml"] diff --git a/examples/pubsub_postgres/README.md b/examples/pubsub_postgres/README.md new file mode 100644 index 000000000..ee1c42aac --- /dev/null +++ b/examples/pubsub_postgres/README.md @@ -0,0 +1,3 @@ +# Postgres Example + +This example requires additional configuration on the Pub/Sub side. Please see https://docs.artie.so/configurations/wip-tutorials/setting-up-pub-sub for further details. diff --git a/examples/pubsub_postgres/application.properties b/examples/pubsub_postgres/application.properties new file mode 100644 index 000000000..d37f970a6 --- /dev/null +++ b/examples/pubsub_postgres/application.properties @@ -0,0 +1,19 @@ +# Offset storage +debezium.source.offset.storage.file.filename=/tmp/foo +debezium.source.offset.flush.interval.ms=0 + +# Pubsub setup: https://debezium.io/documentation/reference/stable/operations/debezium-server.html#_google_cloud_pubsub +debezium.sink.type=pubsub +debezium.sink.pubsub.project.id=PROJECT_ID +debezium.sink.pubsub.ordering.enabled=true + +# Postgres +debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector +debezium.source.database.hostname=postgres +debezium.source.database.port=5432 +debezium.source.database.user=postgres +debezium.source.database.password=postgres +debezium.source.database.dbname=postgres +debezium.source.topic.prefix=dbserver1 +debezium.source.table.include.list=inventory.customers +debezium.source.plugin.name=pgoutput diff --git a/examples/pubsub_postgres/config.yaml b/examples/pubsub_postgres/config.yaml new file mode 100644 index 000000000..8776fd302 --- /dev/null +++ b/examples/pubsub_postgres/config.yaml @@ -0,0 +1,22 @@ +outputSource: test +queue: pubsub + +pubsub: + projectID: artie-labs + pathToCredentials: /tmp/credentials/service-account.json + topicConfigs: + - db: customers + tableName: customers + schema: public + topic: "dbserver1.inventory.customers" + cdcFormat: debezium.postgres.wal2json + cdcKeyFormat: org.apache.kafka.connect.json.JsonConverter + +telemetry: + metrics: + provider: datadog + settings: + tags: + - env:production + namespace: "transfer." + addr: "127.0.0.1:8125" diff --git a/examples/pubsub_postgres/docker-compose.yaml b/examples/pubsub_postgres/docker-compose.yaml new file mode 100644 index 000000000..a3274a690 --- /dev/null +++ b/examples/pubsub_postgres/docker-compose.yaml @@ -0,0 +1,31 @@ +version: '3.9' +services: + postgres: + image: quay.io/debezium/example-postgres:2.0 + ports: + - 5432:5432 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + debezium-server: + image: quay.io/debezium/server:2.0 + container_name: debezium-server + # Sleep the PostgreSQL service to spin up. + command: sh -c "sleep 15 && /debezium/run.sh" + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/credentials/service-account.json + links: + - postgres + ports: + - 8080:8080 + volumes: + - ./application.properties:/debezium/conf/application.properties + - REPLACE_ME:/tmp/credentials/service-account.json + depends_on: + - postgres + transfer: + build: + context: . + dockerfile: Dockerfile + volumes: + - REPLACE_ME:/tmp/credentials/service-account.json diff --git a/examples/pubsub_postgres/tf/main.tf b/examples/pubsub_postgres/tf/main.tf new file mode 100644 index 000000000..37219a051 --- /dev/null +++ b/examples/pubsub_postgres/tf/main.tf @@ -0,0 +1,49 @@ +# In here, we will be doing the following: +# 1) Creating a service account +# 2) Creating a topic +# 3) Creating a subscription +# Note: - We require a topic per table that we want to replicate from. +locals { + project = "PROJECT_ID" + role = "roles/pubsub.editor" +} + +provider "google" { + project = local.project + // Authenticate via gcloud auth application-default login + // This requires the gcloud CLI downloaded: https://cloud.google.com/sdk/docs/install + // Need to enable PubSub API: https://console.cloud.google.com/marketplace/product/google/pubsub.googleapis.com +} + +resource "google_service_account" "artie-svc-account" { + account_id = "artie-service-account" + display_name = "Service Account for Artie Transfer and Debezium" +} + +resource "google_project_iam_member" "transfer" { + project = local.project + role = local.role + member = "serviceAccount:${google_service_account.artie-svc-account.email}" +} + +# Pub/Sub configurations +# We will create the topic and subscription. +resource "google_pubsub_topic" "customer_topic" { + name = "dbserver1.inventory.customers" + project = local.project + + timeouts {} +} + +resource "google_pubsub_subscription" "customer_subscription" { + ack_deadline_seconds = 300 + enable_exactly_once_delivery = false + enable_message_ordering = true + message_retention_duration = "604800s" + name = "transfer_${google_pubsub_topic.customer_topic.name}" + project = local.project + retain_acked_messages = false + topic = google_pubsub_topic.customer_topic.id + + timeouts {} +}