Add in ELK stack (ElasticSearch with additional user setup, Logstash,…

… Kibana), Fluentd, and httpd

README.md

@@ -141,9 +141,10 @@ POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres
 | Data Catalog                | polaris                   | ✅         |
 | Data Catalog                | unitycatalog              | ✅         |
 | Data Catalog                | openmetadata              | ❌         |
+| Data Collector              | fluentd                   | ✅         |
+| Data Collector              | logstash                  | ✅         |
 | Distributed Coordination    | zookeeper                 | ✅         |
 | Distributed Data Processing | flink                     | ✅         |
-| HTTP                        | httpbin                   | ✅         |
 | Identity Management         | keycloak                  | ✅         |
 | Job Orchestrator            | airflow                   | ✅         |
 | Job Orchestrator            | dagster                   | ✅         |
@@ -165,6 +166,8 @@ POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres
 | Real-time OLAP              | pinot                     | ✅         |
 | Schema Registry             | confluent-schema-registry | ✅         |
 | Test Data Management        | data-caterer              | ✅         |
-| Workflow                    | maestro                   | ✅         | 
+| Web Server                  | httpbin                   | ✅         |
+| Web Server                  | httpd                     | ✅         |
+| Workflow                    | maestro                   | ✅         |
 | Workflow                    | temporal                  | ✅         | 
 

data/elasticsearch/data/entrypoint.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+# From ELK repo: https://github.com/deviantony/docker-elk
+
+set -eu
+set -o pipefail
+
+source "${BASH_SOURCE[0]%/*}"/lib.sh
+
+
+# --------------------------------------------------------
+# Users declarations
+
+declare -A users_passwords
+users_passwords=(
+	[logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}"
+	[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
+	[metricbeat_internal]="${METRICBEAT_INTERNAL_PASSWORD:-}"
+	[filebeat_internal]="${FILEBEAT_INTERNAL_PASSWORD:-}"
+	[heartbeat_internal]="${HEARTBEAT_INTERNAL_PASSWORD:-}"
+	[monitoring_internal]="${MONITORING_INTERNAL_PASSWORD:-}"
+	[beats_system]="${BEATS_SYSTEM_PASSWORD=:-}"
+)
+
+declare -A users_roles
+users_roles=(
+	[logstash_internal]='logstash_writer'
+	[metricbeat_internal]='metricbeat_writer'
+	[filebeat_internal]='filebeat_writer'
+	[heartbeat_internal]='heartbeat_writer'
+	[monitoring_internal]='remote_monitoring_collector'
+)
+
+# --------------------------------------------------------
+# Roles declarations
+
+declare -A roles_files
+roles_files=(
+	[logstash_writer]='logstash_writer.json'
+	[metricbeat_writer]='metricbeat_writer.json'
+	[filebeat_writer]='filebeat_writer.json'
+	[heartbeat_writer]='heartbeat_writer.json'
+)
+
+# --------------------------------------------------------
+
+
+log 'Waiting for availability of Elasticsearch. This can take several minutes.'
+
+declare -i exit_code=0
+wait_for_elasticsearch || exit_code=$?
+
+if ((exit_code)); then
+	case $exit_code in
+		6)
+			suberr 'Could not resolve host. Is Elasticsearch running?'
+			;;
+		7)
+			suberr 'Failed to connect to host. Is Elasticsearch healthy?'
+			;;
+		28)
+			suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
+			;;
+		*)
+			suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
+			;;
+	esac
+
+	exit $exit_code
+fi
+
+sublog 'Elasticsearch is running'
+
+log 'Waiting for initialization of built-in users'
+
+wait_for_builtin_users || exit_code=$?
+
+if ((exit_code)); then
+	suberr 'Timed out waiting for condition'
+	exit $exit_code
+fi
+
+sublog 'Built-in users were initialized'
+
+for role in "${!roles_files[@]}"; do
+	log "Role '$role'"
+
+	declare body_file
+	body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
+	if [[ ! -f "${body_file:-}" ]]; then
+		sublog "No role body found at '${body_file}', skipping"
+		continue
+	fi
+
+	sublog 'Creating/updating'
+	ensure_role "$role" "$(<"${body_file}")"
+done
+
+for user in "${!users_passwords[@]}"; do
+	log "User '$user'"
+	if [[ -z "${users_passwords[$user]:-}" ]]; then
+		sublog 'No password defined, skipping'
+		continue
+	fi
+
+	declare -i user_exists=0
+	user_exists="$(check_user_exists "$user")"
+
+	if ((user_exists)); then
+		sublog 'User exists, setting password'
+		set_user_password "$user" "${users_passwords[$user]}"
+	else
+		if [[ -z "${users_roles[$user]:-}" ]]; then
+			suberr '  No role defined, skipping creation'
+			continue
+		fi
+
+		sublog 'User does not exist, creating'
+		create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
+	fi
+done

data/elasticsearch/data/lib.sh

@@ -0,0 +1,242 @@
+#!/usr/bin/env bash
+
+# From ELK repo: https://github.com/deviantony/docker-elk
+
+# Log a message.
+function log {
+	echo "[+] $1"
+}
+
+# Log a message at a sub-level.
+function sublog {
+	echo "   ⠿ $1"
+}
+
+# Log an error.
+function err {
+	echo "[x] $1" >&2
+}
+
+# Log an error at a sub-level.
+function suberr {
+	echo "   ⠍ $1" >&2
+}
+
+# Poll the 'elasticsearch' service until it responds with HTTP code 200.
+function wait_for_elasticsearch {
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" )
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	# retry for max 300s (60*5s)
+	for _ in $(seq 1 60); do
+		local -i exit_code=0
+		output="$(curl "${args[@]}")" || exit_code=$?
+
+		if ((exit_code)); then
+			result=$exit_code
+		fi
+
+		if [[ "${output: -3}" -eq 200 ]]; then
+			result=0
+			break
+		fi
+
+		sleep 5
+	done
+
+	if ((result)) && [[ "${output: -3}" -ne 000 ]]; then
+		echo -e "\n${output::-3}"
+	fi
+
+	return $result
+}
+
+# Poll the Elasticsearch users API until it returns users.
+function wait_for_builtin_users {
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" )
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+
+	local line
+	local -i exit_code
+	local -i num_users
+
+	# retry for max 30s (30*1s)
+	for _ in $(seq 1 30); do
+		num_users=0
+
+		# read exits with a non-zero code if the last read input doesn't end
+		# with a newline character. The printf without newline that follows the
+		# curl command ensures that the final input not only contains curl's
+		# exit code, but causes read to fail so we can capture the return value.
+		# Ref. https://unix.stackexchange.com/a/176703/152409
+		while IFS= read -r line || ! exit_code="$line"; do
+			if [[ "$line" =~ _reserved.+true ]]; then
+				(( num_users++ ))
+			fi
+		done < <(curl "${args[@]}"; printf '%s' "$?")
+
+		if ((exit_code)); then
+			result=$exit_code
+		fi
+
+		# we expect more than just the 'elastic' user in the result
+		if (( num_users > 1 )); then
+			result=0
+			break
+		fi
+
+		sleep 1
+	done
+
+	return $result
+}
+
+# Verify that the given Elasticsearch user exists.
+function check_user_exists {
+	local username=$1
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/user/${username}"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local -i exists=0
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then
+		result=0
+	fi
+	if [[ "${output: -3}" -eq 200 ]]; then
+		exists=1
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}"
+	else
+		echo "$exists"
+	fi
+
+	return $result
+}
+
+# Set password of a given Elasticsearch user.
+function set_user_password {
+	local username=$1
+	local password=$2
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/user/${username}/_password"
+		'-X' 'POST'
+		'-H' 'Content-Type: application/json'
+		'-d' "{\"password\" : \"${password}\"}"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 ]]; then
+		result=0
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}\n"
+	fi
+
+	return $result
+}
+
+# Create the given Elasticsearch user.
+function create_user {
+	local username=$1
+	local password=$2
+	local role=$3
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/user/${username}"
+		'-X' 'POST'
+		'-H' 'Content-Type: application/json'
+		'-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 ]]; then
+		result=0
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}\n"
+	fi
+
+	return $result
+}
+
+# Ensure that the given Elasticsearch role is up-to-date, create it if required.
+function ensure_role {
+	local name=$1
+	local body=$2
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/role/${name}"
+		'-X' 'POST'
+		'-H' 'Content-Type: application/json'
+		'-d' "$body"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 ]]; then
+		result=0
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}\n"
+	fi
+
+	return $result
+}