From 34ca4894c8eac567c8f04eaa3ad2cb6b9193e28e Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vlad@es.net>
Date: Tue, 23 Apr 2024 11:22:13 -0500
Subject: [PATCH 1/5] Merge common parts of compose configs into a single file,
 then override as needed.

---
 .github/workflows/docs.yml                    |  2 +-
 .gitignore                                    |  2 +-
 Makefile                                      | 52 +++++++++---------
 compose.override.local.yml                    | 55 +++++++++++++++++++
 ...ion.yml => compose.override.production.yml | 48 +---------------
 local.yml => compose.yml                      | 52 +++---------------
 6 files changed, 92 insertions(+), 119 deletions(-)
 create mode 100644 compose.override.local.yml
 rename production.yml => compose.override.production.yml (62%)
 rename local.yml => compose.yml (58%)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 6293372e..1e875dc4 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Cache Docker images.
         uses: ScribeMD/docker-cache@0.3.7
         with:
-          key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }}
+          key: docker-${{ runner.os }}-${{ hashFiles('compose*yml') }}
 
       - name: "[pytest]: before"
         run: "./.ci-scripts/pytest_before.sh"
diff --git a/.gitignore b/.gitignore
index ab3f0e51..ed35d0c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -336,7 +336,7 @@ scram/media/
 .ipython/
 .envs/*
 !.envs/.local/
-docker-compose.yaml
+compose.override.yml
 
 coverage.coverage
 coverage.xml
diff --git a/Makefile b/Makefile
index d2ec30c7..23f4a49c 100644
--- a/Makefile
+++ b/Makefile
@@ -3,39 +3,39 @@
 ## toggle-prod: configure make to use the production stack
 .Phony: toggle-prod
 toggle-prod:
-	@ln -sf production.yml docker-compose.yaml
+	@ln -sf compose.override.production.yml compose.override.yml
 
 ## toggle-local: configure make to use the local stack
 .Phony: toggle-local
 toggle-local:
-	@ln -sf local.yml docker-compose.yaml
+	@ln -sf compose.override.local.yml compose.override.yml
 
 # Since toggle-(local|prod) are phony targets, this file is not tracked
 # to compare if its "newer" so running another target with this as a prereq
-# will not run this target again. That would overwrite docker-compose.yaml back to local.yml
+# will not run this target again. That would overwrite compose.yml back to local.yml
 # no matter what, which is bad. Phony targets prevents this
-## docker-compose.yaml: creates file docker-compose.yaml on first run (as a prereq)
-docker-compose.yaml:
-	@ln -sf local.yml docker-compose.yaml
+## compose.yml: creates file compose.yml on first run (as a prereq)
+compose.yml:
+	@ln -sf local.yml compose.yml
 
 ## behave-all: runs behave inside the containers against all of your features
 .Phony: behave-all
-behave-all: docker-compose.yaml
+behave-all: compose.yml
 	@docker compose run django coverage run -a manage.py behave --no-input --simple
 
 ## behave: runs behave inside the containers against a specific feature (append FEATURE=feature_name_here)
 .Phony: behave
-behave: docker-compose.yaml
+behave: compose.yml
 	@docker compose run django python manage.py behave --no-input --simple -i $(FEATURE)
 
 ## behave-translator
 .Phony: behave-translator
-behave-translator: docker-compose.yaml
+behave-translator: compose.yml
 	@docker compose exec -T translator /usr/local/bin/behave /app/acceptance/features
 
 ## build: rebuilds all your containers or a single one if CONTAINER is specified
 .Phony: build
-build: docker-compose.yaml
+build: compose.yml
 	@docker compose up -d --no-deps --build $(CONTAINER)
 	@docker compose restart $(CONTAINER)
 
@@ -50,38 +50,38 @@ ci-test: | toggle-local build migrate run coverage.xml
 
 ## clean: remove local containers and volumes
 .Phony: clean
-clean: docker-compose.yaml
+clean: compose.yml
 	@docker compose rm -f -s
 	@docker volume prune -f
 
 ## collect-static: run collect static admin command
 .Phony: collectstatic
-collectstatic: docker-compose.yaml
+collectstatic: compose.yml
 	@docker compose run django python manage.py collectstatic
 
 ## django-addr: get the IP and ephemeral port assigned to docker:8000
 .Phony: django-addr
-django-addr: docker-compose.yaml
+django-addr: compose.yml
 	@docker compose port django 8000
 
 ## django-url: get the URL based on http://$(make django-addr)
 .Phony: django-url
-django-url: docker-compose.yaml
+django-url: compose.yml
 	@echo http://$$(make django-addr)
 
 ## django-open: open a browser for http://$(make django-addr)
 .Phony: django-open
-django-open: docker-compose.yaml
+django-open: compose.yml
 	@open http://$$(make django-addr)
 
 ## down: turn down docker compose stack
 .Phony: down
-down: docker-compose.yaml
+down: compose.yml
 	@docker compose down
 
 ## exec: executes a given command on a given container (append CONTAINER=container_name_here and COMMAND=command_here)
 .Phony: exec
-exec: docker-compose.yaml
+exec: compose.yml
 	@docker compose exec $(CONTAINER) $(COMMAND)
 
 # This automatically builds the help target based on commands prepended with a double hashbang
@@ -93,42 +93,42 @@ help: Makefile
 # TODO: When we move to flowspec this -a flag with change
 ## list-routes: list gobgp routes
 .Phony: list-routes
-list-routes: docker-compose.yaml
+list-routes: compose.yml
 	@docker compose exec gobgp gobgp global rib -a ipv4
 	@docker compose exec gobgp gobgp global rib -a ipv6
 
 ## migrate: makemigrations and then migrate
 .Phony: migrate
-migrate: docker-compose.yaml
+migrate: compose.yml
 	@docker compose run django python manage.py makemigrations
 	@docker compose run django python manage.py migrate
 
 ## pass-reset: change admin's password
 .Phony: pass-reset
-pass-reset: docker-compose.yaml
+pass-reset: compose.yml
 	@docker compose run django python manage.py changepassword admin
 
 ## pytest: runs pytest inside the containers
 .Phony: pytest
-pytest: docker-compose.yaml
+pytest: compose.yml
 	@docker compose run django coverage run -m pytest
 
-## run: brings up the containers as described in docker-compose.yaml
+## run: brings up the containers as described in compose.yml
 .Phony: run
-run: docker-compose.yaml
+run: compose.yml
 	@docker compose up -d
 
 ## stop: turns off running containers
 .Phony: stop
-stop: docker-compose.yaml
+stop: compose.yml
 	@docker compose stop
 
 ## tail-log: tail a docker container's logs (append CONTAINER=container_name_here)
 .Phony: tail-log
-tail-log: docker-compose.yaml
+tail-log: compose.yml
 	@docker compose logs -f $(CONTAINER)
 
 ## type-check: static type checking
 .Phony: type-check
-type-check: docker-compose.yaml
+type-check: compose.yml
 	@docker compose run django mypy scram
diff --git a/compose.override.local.yml b/compose.override.local.yml
new file mode 100644
index 00000000..3bf03c5d
--- /dev/null
+++ b/compose.override.local.yml
@@ -0,0 +1,55 @@
+---
+version: '3.4'
+
+volumes:
+  local_postgres_data: {}
+  local_postgres_data_backups: {}
+
+services:
+  django:
+    build:
+      dockerfile: ./compose/local/django/Dockerfile
+    image: scram_local_django
+    volumes:
+      - $CI_PROJECT_DIR:/app:z
+      - /tmp/profile_data:/tmp/profile_data
+    env_file:
+      - ./.envs/.local/.django
+      - ./.envs/.local/.postgres
+    ports:
+      - "8000"
+
+  postgres:
+    volumes:
+      - local_postgres_data:/var/lib/postgresql/data:Z
+      - local_postgres_data_backups:/backups:z
+    env_file:
+      - ./.envs/.local/.postgres
+
+  docs:
+    image: scram_local_docs
+    build:
+      context: .
+      dockerfile: ./compose/local/docs/Dockerfile
+    env_file:
+      - ./.envs/.local/.django
+    networks:
+      default: {}
+    volumes:
+      - $CI_PROJECT_DIR/docs:/docs:z
+      - $CI_PROJECT_DIR/config:/app/config:z
+      - $CI_PROJECT_DIR/scram:/app/scram:z
+    ports:
+      - "7000"
+    command: /start-docs
+
+  redis:
+    ports:
+      - "6379"
+
+  gobgp:
+    volumes:
+      - $CI_PROJECT_DIR/gobgp_config:/config:z
+    ports:
+      - "179"
+      - "50051"
diff --git a/production.yml b/compose.override.production.yml
similarity index 62%
rename from production.yml
rename to compose.override.production.yml
index 3a4a1b61..8e99028a 100644
--- a/production.yml
+++ b/compose.override.production.yml
@@ -1,4 +1,4 @@
-version: '3.4'
+---
 
 volumes:
   production_postgres_data: {}
@@ -9,39 +9,14 @@ volumes:
 services:
   django:
     build:
-      context: .
       dockerfile: ./compose/production/django/Dockerfile
     image: scram_production_django
-    depends_on:
-      - postgres
-      - redis
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
-    volumes:
-      - ./staticfiles:/staticfiles
     env_file:
       - ./.envs/.production/.django
       - ./.envs/.production/.postgres
       - /etc/vault.d/secrets/kv_root_security.env
-    command: /start
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://django:5000/process_expired/"]
-      interval: 30s
-      timeout: 30s
-      start_period: 30s
-      retries: 5
 
   postgres:
-    build:
-      context: .
-      dockerfile: ./compose/production/postgres/Dockerfile
-    image: scram_production_postgres
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - production_postgres_data:/var/lib/postgresql/data:Z
       - production_postgres_data_backups:/backups:z
@@ -68,19 +43,12 @@ services:
 
   redis:
     image: redis:5.0
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - production_redis_data:/var/lib/redis:Z
 
   gobgp:
-    image: jauderho/gobgp:v2.32.0
     volumes:
       - ./gobgp_config:/config:z
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     networks:
       default: {}
       peering:
@@ -91,26 +59,12 @@ services:
       - "50051"
 
   translator:
-    build:
-      context: .
-      dockerfile: ./compose/local/translator/Dockerfile
-    depends_on:
-      - redis
-      - gobgp
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     env_file:
       - ./.envs/.production/.translator
 
 networks:
   default:
     enable_ipv6: true
-    ipam:
-      driver: default
-      config:
-        - subnet: 0200:c0:ff:ee::/48
   peering:
     enable_ipv6: true
     driver: macvlan
diff --git a/local.yml b/compose.yml
similarity index 58%
rename from local.yml
rename to compose.yml
index e42895c4..5737fa1b 100644
--- a/local.yml
+++ b/compose.yml
@@ -1,16 +1,10 @@
 ---
 version: '3.4'
 
-volumes:
-  local_postgres_data: {}
-  local_postgres_data_backups: {}
-
 services:
   django:
     build:
       context: .
-      dockerfile: ./compose/local/django/Dockerfile
-    image: scram_local_django
     depends_on:
       postgres:
         condition: service_healthy
@@ -21,13 +15,7 @@ services:
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
     volumes:
-      - $CI_PROJECT_DIR:/app:z
-      - /tmp/profile_data:/tmp/profile_data
-    env_file:
-      - ./.envs/.local/.django
-      - ./.envs/.local/.postgres
-    ports:
-      - "8000"
+      - ./staticfiles:/staticfiles
     command: /start
     healthcheck:
       test: ["CMD", "curl", "-f", "http://django:8000/process_expired/"]
@@ -45,37 +33,15 @@ services:
       default: {}
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
-    volumes:
-      - local_postgres_data:/var/lib/postgresql/data:Z
-      - local_postgres_data_backups:/backups:z
-    env_file:
-      - ./.envs/.local/.postgres
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
 
-  docs:
-    image: scram_local_docs
-    build:
-      context: .
-      dockerfile: ./compose/local/docs/Dockerfile
-    env_file:
-      - ./.envs/.local/.django
-    networks:
-      default: {}
-    volumes:
-      - $CI_PROJECT_DIR/docs:/docs:z
-      - $CI_PROJECT_DIR/config:/app/config:z
-      - $CI_PROJECT_DIR/scram:/app/scram:z
-    ports:
-      - "7000"
-    command: /start-docs
-
   redis:
     image: redis:5.0
+    networks:
+      default: {}
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
-    ports:
-      - "6379"
     healthcheck:
       test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
 
@@ -85,22 +51,20 @@ services:
       default: {}
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
-    volumes:
-      - $CI_PROJECT_DIR/gobgp_config:/config:z
-    ports:
-      - "179"
-      - "50051"
 
   translator:
     build:
       context: .
       dockerfile: ./compose/local/translator/Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      gobgp:
+        condition: service_healthy
     networks:
       default: {}
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
-    depends_on:
-      - gobgp
 
 networks:
   default:

From 32953938a6fbae05c75f504e6bdd8b6900399d00 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vlad@es.net>
Date: Tue, 23 Apr 2024 11:23:43 -0500
Subject: [PATCH 2/5] Re-add gobgp health check

---
 compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compose.yml b/compose.yml
index 5737fa1b..b2aaf4d4 100644
--- a/compose.yml
+++ b/compose.yml
@@ -51,6 +51,8 @@ services:
       default: {}
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
+    healthcheck:
+      test: ["CMD", "gobgp", "global"]
 
   translator:
     build:

From 87f368152a691de489d1ca217daac5592a26027e Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vlad@es.net>
Date: Tue, 23 Apr 2024 11:25:55 -0500
Subject: [PATCH 3/5] Update Makefile logic

---
 Makefile | 55 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/Makefile b/Makefile
index 23f4a49c..4a83c796 100644
--- a/Makefile
+++ b/Makefile
@@ -10,32 +10,33 @@ toggle-prod:
 toggle-local:
 	@ln -sf compose.override.local.yml compose.override.yml
 
-# Since toggle-(local|prod) are phony targets, this file is not tracked
-# to compare if its "newer" so running another target with this as a prereq
-# will not run this target again. That would overwrite compose.yml back to local.yml
-# no matter what, which is bad. Phony targets prevents this
-## compose.yml: creates file compose.yml on first run (as a prereq)
-compose.yml:
-	@ln -sf local.yml compose.yml
+# Since toggle-(local|prod) are phony targets, this file is not
+# tracked to compare if its "newer" so running another target with
+# this as a prereq will not run this target again. That would
+# overwrite compose.override.yml back to compose.override.local.yml no
+# matter what, which is bad. Phony targets prevents this
+## compose.override.yml: creates file compose.override.yml on first run (as a prereq)
+compose.override.yml:
+	@ln -sf compose.override.local.yml compose.override.yml
 
 ## behave-all: runs behave inside the containers against all of your features
 .Phony: behave-all
-behave-all: compose.yml
+behave-all: compose.override.yml
 	@docker compose run django coverage run -a manage.py behave --no-input --simple
 
 ## behave: runs behave inside the containers against a specific feature (append FEATURE=feature_name_here)
 .Phony: behave
-behave: compose.yml
+behave: compose.override.yml
 	@docker compose run django python manage.py behave --no-input --simple -i $(FEATURE)
 
 ## behave-translator
 .Phony: behave-translator
-behave-translator: compose.yml
+behave-translator: compose.override.yml
 	@docker compose exec -T translator /usr/local/bin/behave /app/acceptance/features
 
 ## build: rebuilds all your containers or a single one if CONTAINER is specified
 .Phony: build
-build: compose.yml
+build: compose.override.yml
 	@docker compose up -d --no-deps --build $(CONTAINER)
 	@docker compose restart $(CONTAINER)
 
@@ -50,38 +51,38 @@ ci-test: | toggle-local build migrate run coverage.xml
 
 ## clean: remove local containers and volumes
 .Phony: clean
-clean: compose.yml
+clean: compose.override.yml
 	@docker compose rm -f -s
 	@docker volume prune -f
 
 ## collect-static: run collect static admin command
 .Phony: collectstatic
-collectstatic: compose.yml
+collectstatic: compose.override.yml
 	@docker compose run django python manage.py collectstatic
 
 ## django-addr: get the IP and ephemeral port assigned to docker:8000
 .Phony: django-addr
-django-addr: compose.yml
+django-addr: compose.override.yml
 	@docker compose port django 8000
 
 ## django-url: get the URL based on http://$(make django-addr)
 .Phony: django-url
-django-url: compose.yml
+django-url: compose.override.yml
 	@echo http://$$(make django-addr)
 
 ## django-open: open a browser for http://$(make django-addr)
 .Phony: django-open
-django-open: compose.yml
+django-open: compose.override.yml
 	@open http://$$(make django-addr)
 
 ## down: turn down docker compose stack
 .Phony: down
-down: compose.yml
+down: compose.override.yml
 	@docker compose down
 
 ## exec: executes a given command on a given container (append CONTAINER=container_name_here and COMMAND=command_here)
 .Phony: exec
-exec: compose.yml
+exec: compose.override.yml
 	@docker compose exec $(CONTAINER) $(COMMAND)
 
 # This automatically builds the help target based on commands prepended with a double hashbang
@@ -93,42 +94,42 @@ help: Makefile
 # TODO: When we move to flowspec this -a flag with change
 ## list-routes: list gobgp routes
 .Phony: list-routes
-list-routes: compose.yml
+list-routes: compose.override.yml
 	@docker compose exec gobgp gobgp global rib -a ipv4
 	@docker compose exec gobgp gobgp global rib -a ipv6
 
 ## migrate: makemigrations and then migrate
 .Phony: migrate
-migrate: compose.yml
+migrate: compose.override.yml
 	@docker compose run django python manage.py makemigrations
 	@docker compose run django python manage.py migrate
 
 ## pass-reset: change admin's password
 .Phony: pass-reset
-pass-reset: compose.yml
+pass-reset: compose.override.yml
 	@docker compose run django python manage.py changepassword admin
 
 ## pytest: runs pytest inside the containers
 .Phony: pytest
-pytest: compose.yml
+pytest: compose.override.yml
 	@docker compose run django coverage run -m pytest
 
-## run: brings up the containers as described in compose.yml
+## run: brings up the containers as described in compose.override.yml
 .Phony: run
-run: compose.yml
+run: compose.override.yml
 	@docker compose up -d
 
 ## stop: turns off running containers
 .Phony: stop
-stop: compose.yml
+stop: compose.override.yml
 	@docker compose stop
 
 ## tail-log: tail a docker container's logs (append CONTAINER=container_name_here)
 .Phony: tail-log
-tail-log: compose.yml
+tail-log: compose.override.yml
 	@docker compose logs -f $(CONTAINER)
 
 ## type-check: static type checking
 .Phony: type-check
-type-check: compose.yml
+type-check: compose.override.yml
 	@docker compose run django mypy scram

From 716a226b4d76e0b81c2edaaf520c45f42f55a87d Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vlad@es.net>
Date: Tue, 23 Apr 2024 11:58:47 -0500
Subject: [PATCH 4/5] Restore version, just in case

---
 compose.override.production.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compose.override.production.yml b/compose.override.production.yml
index 8e99028a..0053c270 100644
--- a/compose.override.production.yml
+++ b/compose.override.production.yml
@@ -1,4 +1,5 @@
 ---
+version: '3.4'
 
 volumes:
   production_postgres_data: {}

From de2d9a8449d62a846c42568944d50b82e97120f4 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vlad@es.net>
Date: Tue, 23 Apr 2024 13:00:56 -0500
Subject: [PATCH 5/5] Clean up compose files one last time

---
 compose.override.local.yml      | 9 +++++++++
 compose.override.production.yml | 9 ++++++++-
 compose.yml                     | 9 ---------
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/compose.override.local.yml b/compose.override.local.yml
index 3bf03c5d..bf114584 100644
--- a/compose.override.local.yml
+++ b/compose.override.local.yml
@@ -16,6 +16,8 @@ services:
     env_file:
       - ./.envs/.local/.django
       - ./.envs/.local/.postgres
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://django:8000/process_expired/"]
     ports:
       - "8000"
 
@@ -53,3 +55,10 @@ services:
     ports:
       - "179"
       - "50051"
+
+networks:
+  default:
+    ipam:
+      driver: default
+      config:
+        - subnet: 0200:c0:ff:ee::/48
diff --git a/compose.override.production.yml b/compose.override.production.yml
index 0053c270..eaa97e91 100644
--- a/compose.override.production.yml
+++ b/compose.override.production.yml
@@ -16,6 +16,10 @@ services:
       - ./.envs/.production/.django
       - ./.envs/.production/.postgres
       - /etc/vault.d/secrets/kv_root_security.env
+    volumes:
+      - ./staticfiles:/staticfiles
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://django:5000/process_expired/"]
 
   postgres:
     volumes:
@@ -43,7 +47,6 @@ services:
       - "80:80"
 
   redis:
-    image: redis:5.0
     volumes:
       - production_redis_data:/var/lib/redis:Z
 
@@ -66,6 +69,10 @@ services:
 networks:
   default:
     enable_ipv6: true
+    ipam:
+      driver: default
+      config:
+        - subnet: 0200:c0:ff:ee::/48
   peering:
     enable_ipv6: true
     driver: macvlan
diff --git a/compose.yml b/compose.yml
index b2aaf4d4..30a6c624 100644
--- a/compose.yml
+++ b/compose.yml
@@ -14,11 +14,8 @@ services:
       default: {}
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
-    volumes:
-      - ./staticfiles:/staticfiles
     command: /start
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://django:8000/process_expired/"]
       interval: 30s
       timeout: 30s
       start_period: 30s
@@ -68,9 +65,3 @@ services:
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
 
-networks:
-  default:
-    ipam:
-      driver: default
-      config:
-        - subnet: 0200:c0:ff:ee::/48