Skip to content

Commit

Permalink
chore: add ksql
Browse files Browse the repository at this point in the history
  • Loading branch information
emalfroy authored and ArneD committed Sep 25, 2023
1 parent b72b600 commit b1d6cc6
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 2 deletions.
28 changes: 28 additions & 0 deletions .ksql/CONNECTORS/CONNECTOR_02_POSTAL_OSLO_GEOLOCATION.ksql
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
CREATE SINK CONNECTOR `PostalOsloGeolocationConnector` with (
"topics"= 'postal.snapshot.oslo.flatten.geolocation',
"input.data.format"= 'JSON_SR',
"input.key.format"= 'JSON_SR',
"delete.enabled"= false,
"connector.class"= 'MicrosoftSqlServerSink',
"name"= 'PostalOsloGeolocationConnector',
"kafka.auth.mode"= 'KAFKA_API_KEY',
"kafka.api.key"= '***',
"kafka.api.secret"= '***',
"connection.host"= '***',
"connection.port"= '1433',
"connection.user"= 'geolocation',
"connection.password"= '***',
"db.name"= 'vbr-wms',
"ssl.mode"= 'require',
"insert.mode"= 'UPSERT',
"table.name.format"= 'vbr-wms.geolocation.PostalOsloGeolocation',
"table.types"= 'TABLE',
"db.timezone"= 'UTC',
"pk.mode"= 'record_key',
"pk.fields"= 'msgkey',
"auto.create"= true,
"auto.evolve"= true,
"quote.sql.identifiers"= 'ALWAYS',
"batch.sizes"= 3000,
"tasks.max"= 1
);
4 changes: 2 additions & 2 deletions .ksql/CONNECTORS/CONNECTOR_02_POSTAL_OSLO_GRB.ksql
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ CREATE SINK CONNECTOR `PostalOsloGrbConnector` with (
"connector.class"= 'MicrosoftSqlServerSink',
"name"= 'PostalOsloGrbConnector',
"kafka.auth.mode"= 'KAFKA_API_KEY',
"kafka.api.key"= 'DZKR6A5UYAZDBARE',
"kafka.api.key"= '***',
"kafka.api.secret"= '***',
"connection.host"= 'aiv-beta-grb-basisregisters-db.database.windows.net',
"connection.host"= '***',
"connection.port"= '1433',
"connection.user"= 'GR',
"connection.password"= '***',
Expand Down
10 changes: 10 additions & 0 deletions .ksql/Readme.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
## How to execute per cluster
- put all script `ALL_01_POSTAL_SNAPSHOT_OSLO_STREAM` into ksqlDB
- Set auto.offset.reset = earliest
- execute
- now do the same for the specific script(s)

## How to set up the connectors
- put the connectors script into ksqlDB
- replace *** with the secrets from last pass
- Run the script

0 comments on commit b1d6cc6

Please sign in to comment.