diff --git a/server/2.5-jks/Dockerfile b/server/2.5-jks/Dockerfile new file mode 100644 index 00000000..ffe3d13b --- /dev/null +++ b/server/2.5-jks/Dockerfile @@ -0,0 +1,70 @@ +FROM --platform=linux/amd64 registry.access.redhat.com/ubi8/openjdk-11 + +LABEL maintainer="Debezium Community" + +# +# Set the version, home directory, and MD5 hash. +# +ENV DEBEZIUM_VERSION=2.5.0.Final \ + SERVER_HOME=/debezium \ + MAVEN_REPO_CENTRAL="https://repo1.maven.org/maven2" \ + AWS_MSK_IAM_VERSION=1.1.6 +ENV SERVER_URL_PATH=io/debezium/debezium-server-dist/$DEBEZIUM_VERSION/debezium-server-dist-$DEBEZIUM_VERSION.tar.gz \ + SERVER_MD5=0293ac8e8547bd2a8c7d385ce759682a + +# +# Create a directory for Debezium Server +# +USER root +RUN microdnf -y install gzip && \ + microdnf clean all && \ + mkdir $SERVER_HOME && \ + chmod 755 $SERVER_HOME + +RUN mkdir $SERVER_HOME/conf && \ + mkdir $SERVER_HOME/data + +# +# Download and install Debezium Server +# +RUN curl -fSL -o /tmp/debezium.tar.gz "$MAVEN_REPO_CENTRAL/$SERVER_URL_PATH" + +# +# Verify the contents and then install ... +# +RUN echo "$SERVER_MD5 /tmp/debezium.tar.gz" | md5sum -c - &&\ + tar xzf /tmp/debezium.tar.gz -C $SERVER_HOME --strip-components 1 &&\ + rm -f /tmp/debezium.tar.gz + +## Mount this file onto Debezium +COPY debezium-server-kafka-2.5.0.Final.jar /debezium/lib/ +# +# Download AWS IAM jar and put it in /debezium/lib +# +RUN curl -fSL -o "$SERVER_HOME/lib/aws-msk-iam-auth-all.jar" https://github.com/aws/aws-msk-iam-auth/releases/download/v$AWS_MSK_IAM_VERSION/aws-msk-iam-auth-$AWS_MSK_IAM_VERSION-all.jar + +# +# Now install all the Amazon related certs for DocumentDB TLS +# https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html#w139aac29c11c13b5b9 +# +COPY setup_certs.sh /app/setup_certs.sh +RUN microdnf -y install curl openssl perl +RUN chmod +x /app/setup_certs.sh +RUN mkdir -p /tmp/certs +RUN /app/setup_certs.sh + +# +# Allow random UID to use Debezium Server +# +RUN chmod -R g+w,o+w $SERVER_HOME + +# Set the working directory to the Debezium Server home directory +WORKDIR $SERVER_HOME + +# +# Expose the ports and set up volumes for the data, transaction log, and configuration +# +EXPOSE 8080 +VOLUME ["/debezium/conf","/debezium/data"] + +CMD ["/debezium/run.sh"] diff --git a/server/2.5-jks/Dockerfile.local b/server/2.5-jks/Dockerfile.local new file mode 100644 index 00000000..6741cdb5 --- /dev/null +++ b/server/2.5-jks/Dockerfile.local @@ -0,0 +1,53 @@ +FROM registry.access.redhat.com/ubi8/openjdk-11 + +LABEL maintainer="Debezium Community" + +# +# Set the version, home directory, and MD5 hash. +# +ENV SERVER_HOME=/debezium + +# +# Create a directory for Debezium Server +# +USER root +RUN microdnf -y install gzip && \ + microdnf clean all && \ + mkdir $SERVER_HOME && \ + chmod 755 $SERVER_HOME + +# +# Change ownership and switch user +# +RUN chown -R jboss $SERVER_HOME && \ + chgrp -R jboss $SERVER_HOME +USER jboss + +RUN mkdir $SERVER_HOME/conf && \ + mkdir $SERVER_HOME/data + +# +# Download and install Debezium Server +# +COPY debezium-server-dist-2.2.0-SNAPSHOT.tar.gz /tmp/debezium.tar.gz + +# +# Verify the contents and then install ... +# +RUN tar xzf /tmp/debezium.tar.gz -C $SERVER_HOME --strip-components 1 + +# +# Allow random UID to use Debezium Server +# +RUN chmod -R g+w,o+w $SERVER_HOME + +# Set the working directory to the Debezium Server home directory +WORKDIR $SERVER_HOME + +# +# Expose the ports and set up volumes for the data, transaction log, and configuration +# +EXPOSE 8080 +VOLUME ["/debezium/conf","/debezium/data"] + +CMD ["/debezium/run.sh"] diff --git a/server/2.5-jks/README.md b/server/2.5-jks/README.md new file mode 100644 index 00000000..4bc53b3f --- /dev/null +++ b/server/2.5-jks/README.md @@ -0,0 +1,104 @@ +# What is Debezium? + +Debezium is a distributed platform that turns your existing databases into event streams, so applications can quickly react to each row-level change in the databases. + +# What is Debezium Server? + +Debezium can be deployed either as connector instances in a [Kafka Connect](https://kafka.apache.org/documentation/#connect) cluster, or as a standalone application - Debezium Server. +Debezium [Server](https://debezium.io/documentation/reference/operations/debezium-server.html) is a [Quarkus-based](https://quarkus.io/) high-performance application that streams data from database to a one of supported sinks or a user developed sink. + +Debezium Server supports multiple converters to provide different output message formats. + + +# How to use this image + +The image requires as a dependency source and sink systems to read data from and write output messages to. + +The application itself can be configured either via environment variables or via `appliaction.properties` injected into the container via a volume. + +Starting an instance of Debezium Server using this container image is simple: + + $ docker run -it --name debezium -p 8080:8080 -v $PWD/conf:/debezium/conf -v $PWD/data:/debezium/data quay.io/debezium/server + + +## Example + +If you want to try the image yourself then please follow the steps to establish the necessary environment. + +Start PostgreSQL source database: + + $ docker run -d --name postgres -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres quay.io/debezium/example-postgres + +Start Apache Pulsar sink: + + $ docker run -d --name pulsar -p 6650:6650 -p 7080:8080 apachepulsar/pulsar:2.5.2 bin/pulsar standalone + +Wait for Pulsar sink to start: + + $ docker logs -f pulsar + +Prepare Debezium Server deployment: + +``` + $ mkdir {data,conf}; chmod 777 {data,conf} + $ cat <<-EOF > conf/application.properties +debezium.sink.type=pulsar +debezium.sink.pulsar.client.serviceUrl=pulsar://pulsar:6650 +debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector +debezium.source.offset.storage.file.filename=data/offsets.dat +debezium.source.offset.flush.interval.ms=0 +debezium.source.database.hostname=postgres +debezium.source.database.port=5432 +debezium.source.database.user=postgres +debezium.source.database.password=postgres +debezium.source.database.dbname=postgres +debezium.source.topic.prefix=tutorial +debezium.source.schema.include.list=inventory +debezium.source.plugin.name=pgoutput +EOF +``` + +Note that the configuration file can be replaced with environment variables where every property translates to uppercase and dots are replaced with underscore, e.g. `debezium.sink.type` becomes `DEBEZIUM_SINK_TYPE`. + +Start the Debezium Server: + + $ docker run -it --name debezium -p 8080:8080 -v $PWD/conf:/debezium/conf -v $PWD/data:/debezium/data --link postgres --link pulsar quay.io/debezium/server + + +# Environment variables + +The Debezium Server image uses several environment variables to configure JVM and source/sink when running this image. + + +### `JAVA_OPTS` + +This environment variable is passed to command line when `java` command is invoked. +It could be used to tune memory settings etc. + +### `DEBEZIUM_OPTS` + +This environment variable is used in the same way as `JAVA_OPTS` and servers only for logical separation of Debezium Server specific settings. + +### Source/sink Configuration options + +All configuration options that could be present in `application.properties` can be either added or overridden via environment variables. +This is enabled by using [MicroProfile Config](https://github.com/eclipse/microprofile-config) in Debezium Server. + + + +# Ports + +Containers created using this image will expose port `8080`, which is the standard port to access [MicroProfile Health](https://github.com/eclipse/microprofile-health) endpoint. + + +# Volumes + +The container image exposes two volumes: + +### `/debezium/conf` + +In this volume the configuration files (mostly `application.properties`) are located. + +### `/debezium/data` + +In this volume the data files (mostly file offset storage) are located. diff --git a/server/2.5-jks/debezium-server-kafka-2.5.0.Final.jar b/server/2.5-jks/debezium-server-kafka-2.5.0.Final.jar new file mode 100644 index 00000000..60534d1d Binary files /dev/null and b/server/2.5-jks/debezium-server-kafka-2.5.0.Final.jar differ diff --git a/server/2.5-jks/setup_certs.sh b/server/2.5-jks/setup_certs.sh new file mode 100644 index 00000000..aa22fdd1 --- /dev/null +++ b/server/2.5-jks/setup_certs.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# IMPORTANT: This script is copied from: https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html#w139aac29c11c13b5b9 +# and modified to inject the certificates into Java's default cacert +mydir=/tmp/certs +truststore=$JAVA_HOME/lib/security/cacerts +storepassword=changeit + +curl -sS "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem" > ${mydir}/global-bundle.pem +awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n ".pem"}' < ${mydir}/global-bundle.pem + +for CERT in rds-ca-*; do + alias=$(openssl x509 -noout -text -in $CERT | perl -ne 'next unless /Subject:/; s/.*(CN=|CN = )//; print') + echo "Importing $alias" + keytool -trustcacerts -import -file ${CERT} -alias "${alias}" -storepass ${storepassword} -keystore ${truststore} -noprompt + rm $CERT +done diff --git a/server/2.5/Dockerfile b/server/2.5/Dockerfile index fe86610d..57d20c59 100644 --- a/server/2.5/Dockerfile +++ b/server/2.5/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi8/openjdk-11 +FROM --platform=linux/amd64 registry.access.redhat.com/ubi8/openjdk-11 LABEL maintainer="Debezium Community" @@ -7,7 +7,8 @@ LABEL maintainer="Debezium Community" # ENV DEBEZIUM_VERSION=2.5.4.Final \ SERVER_HOME=/debezium \ - MAVEN_REPO_CENTRAL="https://repo1.maven.org/maven2" + MAVEN_REPO_CENTRAL="https://repo1.maven.org/maven2" \ + AWS_MSK_IAM_VERSION=1.1.6 ENV SERVER_URL_PATH=io/debezium/debezium-server-dist/$DEBEZIUM_VERSION/debezium-server-dist-$DEBEZIUM_VERSION.tar.gz \ SERVER_MD5=d75ba8c21e1d14f320b79d92107278b0 @@ -20,13 +21,6 @@ RUN microdnf -y install gzip && \ mkdir $SERVER_HOME && \ chmod 755 $SERVER_HOME -# -# Change ownership and switch user -# -RUN chown -R jboss $SERVER_HOME && \ - chgrp -R jboss $SERVER_HOME -USER jboss - RUN mkdir $SERVER_HOME/conf && \ mkdir $SERVER_HOME/data @@ -42,6 +36,13 @@ RUN echo "$SERVER_MD5 /tmp/debezium.tar.gz" | md5sum -c - &&\ tar xzf /tmp/debezium.tar.gz -C $SERVER_HOME --strip-components 1 &&\ rm -f /tmp/debezium.tar.gz +## Mount this file onto Debezium +COPY debezium-server-kafka-2.5.0.Final.jar /debezium/lib/ +# +# Download AWS IAM jar and put it in /debezium/lib +# +RUN curl -fSL -o "$SERVER_HOME/lib/aws-msk-iam-auth-all.jar" https://github.com/aws/aws-msk-iam-auth/releases/download/v$AWS_MSK_IAM_VERSION/aws-msk-iam-auth-$AWS_MSK_IAM_VERSION-all.jar + # # Allow random UID to use Debezium Server # diff --git a/server/2.5/debezium-server-kafka-2.5.0.Final.jar b/server/2.5/debezium-server-kafka-2.5.0.Final.jar new file mode 100644 index 00000000..60534d1d Binary files /dev/null and b/server/2.5/debezium-server-kafka-2.5.0.Final.jar differ diff --git a/server/specific_version.sh b/server/specific_version.sh new file mode 100755 index 00000000..35ea4f68 --- /dev/null +++ b/server/specific_version.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Which debezium-server would you like to build and push? (e.g: 2.5)" +read VERSION +docker build -t artielabs/debezium-server:$VERSION $VERSION/ +docker push artielabs/debezium-server:$VERSION