Snowplow docker containers should take custom environment variables f…

…rom an optional file (close #398)
snowplow · Feb 9, 2024 · 4c00c6e · 4c00c6e
1 parent 7ee3bdc
commit 4c00c6e
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 1 deletion.
diff --git a/provisioning/resources/configs/compositions/docker-compose-aws.yml b/provisioning/resources/configs/compositions/docker-compose-aws.yml
@@ -67,6 +67,7 @@ services:
         awslogs-stream: "elasticsearch-loader-good"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/es-loader-good.env
     mem_limit: ${SP_ES_LOADER_MEM_SIZE}
 
   elasticsearch-loader-bad:
@@ -85,6 +86,7 @@ services:
         awslogs-stream: "elasticsearch-loader-bad"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/es-loader-bad.env
     mem_limit: ${SP_ES_LOADER_MEM_SIZE}
 
   nsqlookupd:
@@ -155,6 +157,7 @@ services:
         awslogs-stream: "scala-stream-collector"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/collector.env
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
   enrich:
@@ -177,6 +180,7 @@ services:
         awslogs-stream: "enrich"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/enrich.env
     mem_limit: ${SP_ENRICH_MEM_SIZE}
 
   postgres:

diff --git a/provisioning/resources/configs/compositions/docker-compose-gcp.yml b/provisioning/resources/configs/compositions/docker-compose-gcp.yml
@@ -58,6 +58,7 @@ services:
       driver: gcplogs
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/es-loader-good.env
     mem_limit: ${SP_ES_LOADER_MEM_SIZE}
 
   elasticsearch-loader-bad:
@@ -73,6 +74,7 @@ services:
       driver: gcplogs
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/es-loader-bad.env
     mem_limit: ${SP_ES_LOADER_MEM_SIZE}
 
   nsqlookupd:
@@ -131,6 +133,7 @@ services:
       driver: gcplogs
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/collector.env
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
   enrich:
@@ -150,6 +153,7 @@ services:
       driver: gcplogs
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/enrich.env
     mem_limit: ${SP_ENRICH_MEM_SIZE}
 
   postgres:

diff --git a/provisioning/roles/common_vars.yml b/provisioning/roles/common_vars.yml
@@ -10,4 +10,5 @@ init_dir:             "{{main_dir}}/init"
 ui_dir:               "{{main_dir}}/ui/dist"
 ansible_cache_dir:    "/var/cache/ansible"
 software_install_dir: "/usr/local/share"
-user_binaries:        "/usr/local/bin"
+user_binaries:        "/usr/local/bin"
+envs_dir:             "{{main_dir}}/envs"
diff --git a/provisioning/roles/docker/files/docker-compose.yml b/provisioning/roles/docker/files/docker-compose.yml
@@ -64,6 +64,7 @@ services:
         max-file: "10"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/es-loader-good.env
     mem_limit: ${SP_ES_LOADER_MEM_SIZE}
 
   elasticsearch-loader-bad:
@@ -81,6 +82,7 @@ services:
         max-file: "10"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/es-loader-bad.env
     mem_limit: ${SP_ES_LOADER_MEM_SIZE}
 
   nsqlookupd:
@@ -147,6 +149,7 @@ services:
         max-file: "10"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/collector.env
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
   enrich:
@@ -168,6 +171,7 @@ services:
         max-file: "10"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
+    env_file: /home/ubuntu/snowplow/envs/enrich.env
     mem_limit: ${SP_ENRICH_MEM_SIZE}
 
   postgres:

diff --git a/provisioning/roles/sp_mini_1_create_dirs/tasks/main.yml b/provisioning/roles/sp_mini_1_create_dirs/tasks/main.yml
@@ -21,6 +21,7 @@
     - "{{init_dir}}"
     - "{{ui_dir}}"
     - "{{nsq_data_dir}}"
+    - "{{envs_dir}}"
 
 #playbook_dir is ansible defined variable which stores top level ansible-playbook directory
 - name: Copy folders to the /home/ubuntu/snowplow

diff --git a/provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml b/provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml
@@ -38,3 +38,15 @@
 - name: Set owner of the main directory
   become: yes
   shell: "chown -R ubuntu:ubuntu {{home_dir}} && chmod 755 -R {{home_dir}}"
+
+- name: Create empty env files
+  become: yes
+  file:
+    state: touch
+    path: "{{ envs_dir }}/{{ item }}"
+  loop:
+    - es-loader-good.env
+    - es-loader-bad.env
+    - enrich.env
+    - collector.env
+