Merge pull request #38 from esnet-security/topic/vlad/compose_files_m…

…erge Merge common parts of compose configs into a single file, then override as needed
esnet-security · Apr 23, 2024 · 933754a · 933754a
2 parents bbbb90e + de2d9a8
commit 933754a
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 122 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Cache Docker images.
         uses: ScribeMD/[email protected]
         with:
-          key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }}
+          key: docker-${{ runner.os }}-${{ hashFiles('compose*yml') }}
 
       - name: "[pytest]: before"
         run: "./.ci-scripts/pytest_before.sh"

diff --git a/.gitignore b/.gitignore
@@ -336,7 +336,7 @@ scram/media/
 .ipython/
 .envs/*
 !.envs/.local/
-docker-compose.yaml
+compose.override.yml
 
 coverage.coverage
 coverage.xml
diff --git a/Makefile b/Makefile
@@ -3,39 +3,40 @@
 ## toggle-prod: configure make to use the production stack
 .Phony: toggle-prod
 toggle-prod:
-	@ln -sf production.yml docker-compose.yaml
+	@ln -sf compose.override.production.yml compose.override.yml
 
 ## toggle-local: configure make to use the local stack
 .Phony: toggle-local
 toggle-local:
-	@ln -sf local.yml docker-compose.yaml
+	@ln -sf compose.override.local.yml compose.override.yml
 
-# Since toggle-(local|prod) are phony targets, this file is not tracked
-# to compare if its "newer" so running another target with this as a prereq
-# will not run this target again. That would overwrite docker-compose.yaml back to local.yml
-# no matter what, which is bad. Phony targets prevents this
-## docker-compose.yaml: creates file docker-compose.yaml on first run (as a prereq)
-docker-compose.yaml:
-	@ln -sf local.yml docker-compose.yaml
+# Since toggle-(local|prod) are phony targets, this file is not
+# tracked to compare if its "newer" so running another target with
+# this as a prereq will not run this target again. That would
+# overwrite compose.override.yml back to compose.override.local.yml no
+# matter what, which is bad. Phony targets prevents this
+## compose.override.yml: creates file compose.override.yml on first run (as a prereq)
+compose.override.yml:
+	@ln -sf compose.override.local.yml compose.override.yml
 
 ## behave-all: runs behave inside the containers against all of your features
 .Phony: behave-all
-behave-all: docker-compose.yaml
+behave-all: compose.override.yml
 	@docker compose run django coverage run -a manage.py behave --no-input --simple
 
 ## behave: runs behave inside the containers against a specific feature (append FEATURE=feature_name_here)
 .Phony: behave
-behave: docker-compose.yaml
+behave: compose.override.yml
 	@docker compose run django python manage.py behave --no-input --simple -i $(FEATURE)
 
 ## behave-translator
 .Phony: behave-translator
-behave-translator: docker-compose.yaml
+behave-translator: compose.override.yml
 	@docker compose exec -T translator /usr/local/bin/behave /app/acceptance/features
 
 ## build: rebuilds all your containers or a single one if CONTAINER is specified
 .Phony: build
-build: docker-compose.yaml
+build: compose.override.yml
 	@docker compose up -d --no-deps --build $(CONTAINER)
 	@docker compose restart $(CONTAINER)
 
@@ -50,38 +51,38 @@ ci-test: | toggle-local build migrate run coverage.xml
 
 ## clean: remove local containers and volumes
 .Phony: clean
-clean: docker-compose.yaml
+clean: compose.override.yml
 	@docker compose rm -f -s
 	@docker volume prune -f
 
 ## collect-static: run collect static admin command
 .Phony: collectstatic
-collectstatic: docker-compose.yaml
+collectstatic: compose.override.yml
 	@docker compose run django python manage.py collectstatic
 
 ## django-addr: get the IP and ephemeral port assigned to docker:8000
 .Phony: django-addr
-django-addr: docker-compose.yaml
+django-addr: compose.override.yml
 	@docker compose port django 8000
 
 ## django-url: get the URL based on http://$(make django-addr)
 .Phony: django-url
-django-url: docker-compose.yaml
+django-url: compose.override.yml
 	@echo http://$$(make django-addr)
 
 ## django-open: open a browser for http://$(make django-addr)
 .Phony: django-open
-django-open: docker-compose.yaml
+django-open: compose.override.yml
 	@open http://$$(make django-addr)
 
 ## down: turn down docker compose stack
 .Phony: down
-down: docker-compose.yaml
+down: compose.override.yml
 	@docker compose down
 
 ## exec: executes a given command on a given container (append CONTAINER=container_name_here and COMMAND=command_here)
 .Phony: exec
-exec: docker-compose.yaml
+exec: compose.override.yml
 	@docker compose exec $(CONTAINER) $(COMMAND)
 
 # This automatically builds the help target based on commands prepended with a double hashbang
@@ -93,42 +94,42 @@ help: Makefile
 # TODO: When we move to flowspec this -a flag with change
 ## list-routes: list gobgp routes
 .Phony: list-routes
-list-routes: docker-compose.yaml
+list-routes: compose.override.yml
 	@docker compose exec gobgp gobgp global rib -a ipv4
 	@docker compose exec gobgp gobgp global rib -a ipv6
 
 ## migrate: makemigrations and then migrate
 .Phony: migrate
-migrate: docker-compose.yaml
+migrate: compose.override.yml
 	@docker compose run django python manage.py makemigrations
 	@docker compose run django python manage.py migrate
 
 ## pass-reset: change admin's password
 .Phony: pass-reset
-pass-reset: docker-compose.yaml
+pass-reset: compose.override.yml
 	@docker compose run django python manage.py changepassword admin
 
 ## pytest: runs pytest inside the containers
 .Phony: pytest
-pytest: docker-compose.yaml
+pytest: compose.override.yml
 	@docker compose run django coverage run -m pytest
 
-## run: brings up the containers as described in docker-compose.yaml
+## run: brings up the containers as described in compose.override.yml
 .Phony: run
-run: docker-compose.yaml
+run: compose.override.yml
 	@docker compose up -d
 
 ## stop: turns off running containers
 .Phony: stop
-stop: docker-compose.yaml
+stop: compose.override.yml
 	@docker compose stop
 
 ## tail-log: tail a docker container's logs (append CONTAINER=container_name_here)
 .Phony: tail-log
-tail-log: docker-compose.yaml
+tail-log: compose.override.yml
 	@docker compose logs -f $(CONTAINER)
 
 ## type-check: static type checking
 .Phony: type-check
-type-check: docker-compose.yaml
+type-check: compose.override.yml
 	@docker compose run django mypy scram
diff --git a/local.yml → compose.override.local.yml b/local.yml → compose.override.local.yml
@@ -8,50 +8,25 @@ volumes:
 services:
   django:
     build:
-      context: .
       dockerfile: ./compose/local/django/Dockerfile
     image: scram_local_django
-    depends_on:
-      postgres:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - $CI_PROJECT_DIR:/app:z
       - /tmp/profile_data:/tmp/profile_data
     env_file:
       - ./.envs/.local/.django
       - ./.envs/.local/.postgres
-    ports:
-      - "8000"
-    command: /start
     healthcheck:
       test: ["CMD", "curl", "-f", "http://django:8000/process_expired/"]
-      interval: 30s
-      timeout: 30s
-      start_period: 30s
-      retries: 5
+    ports:
+      - "8000"
 
   postgres:
-    build:
-      context: .
-      dockerfile: ./compose/production/postgres/Dockerfile
-    image: scram_production_postgres
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - local_postgres_data:/var/lib/postgresql/data:Z
       - local_postgres_data_backups:/backups:z
     env_file:
       - ./.envs/.local/.postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
 
   docs:
     image: scram_local_docs
@@ -71,38 +46,15 @@ services:
     command: /start-docs
 
   redis:
-    image: redis:5.0
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     ports:
       - "6379"
-    healthcheck:
-      test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
 
   gobgp:
-    image: jauderho/gobgp:v2.32.0
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - $CI_PROJECT_DIR/gobgp_config:/config:z
     ports:
       - "179"
       - "50051"
-    healthcheck:
-      test: ["CMD", "gobgp", "global"]
-
-  translator:
-    build:
-      context: .
-      dockerfile: ./compose/local/translator/Dockerfile
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
-    depends_on:
-      - gobgp
 
 networks:
   default:

diff --git a/production.yml → compose.override.production.yml b/production.yml → compose.override.production.yml
@@ -1,3 +1,4 @@
+---
 version: '3.4'
 
 volumes:
@@ -9,39 +10,18 @@ volumes:
 services:
   django:
     build:
-      context: .
       dockerfile: ./compose/production/django/Dockerfile
     image: scram_production_django
-    depends_on:
-      - postgres
-      - redis
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
-    volumes:
-      - ./staticfiles:/staticfiles
     env_file:
       - ./.envs/.production/.django
       - ./.envs/.production/.postgres
       - /etc/vault.d/secrets/kv_root_security.env
-    command: /start
+    volumes:
+      - ./staticfiles:/staticfiles
     healthcheck:
       test: ["CMD", "curl", "-f", "http://django:5000/process_expired/"]
-      interval: 30s
-      timeout: 30s
-      start_period: 30s
-      retries: 5
 
   postgres:
-    build:
-      context: .
-      dockerfile: ./compose/production/postgres/Dockerfile
-    image: scram_production_postgres
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - production_postgres_data:/var/lib/postgresql/data:Z
       - production_postgres_data_backups:/backups:z
@@ -67,20 +47,12 @@ services:
       - "80:80"
 
   redis:
-    image: redis:5.0
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     volumes:
       - production_redis_data:/var/lib/redis:Z
 
   gobgp:
-    image: jauderho/gobgp:v2.32.0
     volumes:
       - ./gobgp_config:/config:z
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     networks:
       default: {}
       peering:
@@ -91,16 +63,6 @@ services:
       - "50051"
 
   translator:
-    build:
-      context: .
-      dockerfile: ./compose/local/translator/Dockerfile
-    depends_on:
-      - redis
-      - gobgp
-    networks:
-      default: {}
-    sysctls:
-      - net.ipv6.conf.all.disable_ipv6=0
     env_file:
       - ./.envs/.production/.translator