task/WG-134 setup docker compose for geoapi services vm (#142)

* Add initial/work-in-progress docker-compose

* Add nginx conf

* Move docker infrastructure to devops folder

* Move remaining docker infrastructure to devops folder

* Fix local development's docker-compose

* Add additional improvements to docker-compose for geoapi-services vms

* Add some variables

* Use one env variable for mapillary

* Fix dockerfiles

* Add secret.env to gitignore

* Update docker-compose test yaml

* Fix installing of python packages on github actions workflow

* Change to devops/ before running test on CI workflow

* Simplify install/pytest step in CI workflow

* Refactor location of dockerfiles and images used

* Fix nginx conf for deployment

* Change secrets location to /opt/portal/conf/secret.env

* Use absolute path for secrets.env

* Add makefile for geoapi-services

* Refactor dockerfile

* Correct path to poetry.lock and pyproject.toml

* Fix paths in dockerfile

* Rename Dockerfile.potree to Dockerfile.worker

* Rename Dockerfile.potree to Dockerfile.worker again

* Fix workflow

* Fix workflow

* Remove env file usage in Makefile

Using docker-compose.yml where that is hardcoded

* Pass in GEOAPI_TAG from env

* Remove echo

* Fix docker compose statement

* update docker compose and Makefile

* Fix MAKEFILE

* Add todo

* Fix MAKEFILE again

* Add missing RABBITMQ_VHOST

* Do not run docker up in background for testing

* Fix gitignore and improve dockerignore

* Bump rabbitmq version

* Update todo

* Fix tabbitmq password variable on example secrets file

* Refactor how settings are set based on APP_ENV

* Add hazmapper to docker-compose

* Fix makefile

* Expose rabbitmq port from geoapi-services

* Improve make build command

* Improve deploy command

* Add a copy of poetry files

* Fix unit test settings

* Move to devops before running test

* Removed unused steps in workflow

.dockerignore

@@ -1,4 +1,6 @@
 **.pyc
 .git
 **/__pycache__
-
+# Secret env file
+.env
+*secrets.env

.github/workflows/main.yml

@@ -58,9 +58,13 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-poetry-
     - name: Install Python Packages
-      run: poetry install
+      run: |
+        cd devops
+        poetry install
     - name: Run server-side unit tests
-      run: poetry run pytest
+      run: |
+        cd devops
+        poetry run pytest ../geoapi
   Workers_Unit_Tests:
     runs-on: ubuntu-latest
     services:
@@ -86,7 +90,7 @@ jobs:
     - name: Build worker image
       run: |
         docker pull taccaci/geoapi-workers:latest
-        docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f Dockerfile.potree .
+        docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.worker .
     - name: Run worker test
       run: |
         docker run --network="host" -e APP_ENV='testing' -e DB_HOST='localhost'  taccaci/geoapi-workers:latest pytest -m "worker"

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 
 # Secret env file
 .env
+*secrets.env
 
 # C extensions
 *.so

Makefile

@@ -2,15 +2,34 @@ TAG := $(shell git log --format=%h -1)
 GEOAPI_IMAGE=taccaci/geoapi
 GEOAPI_WORKERS=taccaci/geoapi-workers
 
+.PHONY: start
+start:
+	docker-compose -f devops/docker-compose.local.yml --env-file .env up
+
+.PHONY: stop
+stop:
+	docker-compose -f devops/docker-compose.local.yml --env-file .env down
+
+.PHONY: build
+build:
+	make geoapi && make workers
+
 .PHONY: geoapi
 geoapi:
-	docker build -t $(GEOAPI_IMAGE):$(TAG) .
+	docker build -t $(GEOAPI_IMAGE):$(TAG) -f devops/Dockerfile .
 	docker tag $(GEOAPI_IMAGE):$(TAG) $(GEOAPI_IMAGE):latest
+	docker tag $(GEOAPI_IMAGE):$(TAG) $(GEOAPI_IMAGE):local
 
 .PHONY: workers
 workers:
-	docker build -t $(GEOAPI_WORKERS):$(TAG) -f Dockerfile.potree .
+	docker build -t $(GEOAPI_WORKERS):$(TAG) -f devops/Dockerfile.worker .
 	docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):latest
+	docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):local
+
+
+.PHONY: deploy
+deploy:
+	make deploy-geoapi && make deploy-workers
 
 .PHONY: deploy-geoapi
 deploy-geoapi:

README.md

@@ -24,7 +24,8 @@ information. An .env file for developers can be found on [UT Stache](https://sta
 The API is built with flask and flask-restplus. It is running in its own container
 under gunicorn on port 8000
 
-`docker-compose up`
+`make build`
+`make start`
 
 ###### Initialize the database
 
@@ -87,17 +88,17 @@ alembic revision --autogenerate
 
 Run route/service tests on the `api` container
 ```
-docker-compose -f docker-compose.test.yml -p geoapi_test run api pytest
+docker-compose -f devops/docker-compose.test.yml -p geoapi_test run api pytest
 ```
 
 Run worker-related tasks on the `workers` container
 ```
-docker-compose -f docker-compose.test.yml -p geoapi_test run workers pytest -m "worker"
+docker-compose -f devops/docker-compose.test.yml -p geoapi_test run workers pytest -m "worker"
 ```
 
 Note that images need to be rebuilt before running tests if they have been updated (e.g. packages):
 ```
-docker-compose -f docker-compose.test.yml build
+make build
 ```
 ## Kubernetes (Production/Staging)
 

Dockerfile → devops/Dockerfile

@@ -12,10 +12,13 @@ ENV POETRY_HOME=/opt/poetry
 ENV PATH="$POETRY_HOME/bin:$PATH"
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN poetry config virtualenvs.create false
-COPY pyproject.toml poetry.lock ./
-RUN poetry install
 
 RUN mkdir /app
-COPY ./geoapi /app/geoapi
+WORKDIR /app
+
+COPY devops/poetry.lock devops/pyproject.toml ./
+RUN poetry install
+
+COPY geoapi /app/geoapi
 ENV PYTHONPATH "${PYTHONPATH}:/app"
 WORKDIR /app/geoapi

Dockerfile.potree → devops/Dockerfile.worker

@@ -21,7 +21,7 @@ RUN git clone -b develop https://github.com/potree/PotreeConverter.git && cd Pot
 mkdir build && cd build && \
 cmake -DCMAKE_BUILD_TYPE=Release -DLASZIP_INCLUDE_DIRS=/opt/LAStools/LASzip/dll/ -DLASZIP_LIBRARY=/usr/local/lib/liblaszip.so .. && \
 make && make install && cp -r /opt/PotreeConverter/PotreeConverter/resources /resources
-ADD misc/potree/page_template /resources/page_template
+ADD devops/misc/potree/page_template /resources/page_template
 
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2
 
@@ -32,16 +32,17 @@ ENV POETRY_HOME=/opt/poetry
 ENV PATH="$POETRY_HOME/bin:$PATH"
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN poetry config virtualenvs.create false
-COPY pyproject.toml poetry.lock ./
+COPY devops/pyproject.toml devops/poetry.lock ./
+
 RUN poetry install
 
 ENV PYTHONPATH "${PYTHONPATH}:/app"
 
-# isntall geos into condo the base pdal image is using
+# install geos into condo the base pdal image is using
 RUN conda install geos -y -n base
 
 
 WORKDIR /
 RUN mkdir app
-COPY ./geoapi /app/geoapi
+COPY geoapi /app/geoapi
 WORKDIR /app/geoapi

docker-compose.yml → devops/docker-compose.local.yml

@@ -11,7 +11,7 @@ networks:
 services:
 
   rabbitmq:
-    image: rabbitmq
+    image: rabbitmq:3.12.6
     networks:
       - geoapi
     environment:
@@ -38,40 +38,36 @@ services:
       - 8888:80
     volumes:
       - assets:/assets
-      - ./conf/nginx.conf:/etc/nginx/nginx.conf
+      - ./local_conf/nginx.conf:/etc/nginx/nginx.conf
     networks:
       - geoapi
 
 
   workers:
-    build:
-      context: .
-      dockerfile: ./Dockerfile.potree
+    image: taccaci/geoapi-workers:local
     networks:
       - geoapi
     volumes:
-      - .:/app
+      - ../:/app
       - assets:/assets
     environment:
-      - MAPILLARY_CLIENT_TOKEN=4866220476802272
+      - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN}
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=development
       - ASSETS_BASE_DIR=/assets
-      - TENANT
+      - TENANT=${TENANT}
     stdin_open: true
     tty: true
     container_name: geoapiworkers
     hostname: geoapiworkers
     command: "celery -A geoapi.celery_app worker -l info"
 
   celerybeat:
-    build:
-      context: .
-      dockerfile: ./Dockerfile.potree
+    image: taccaci/geoapi-workers:local
     networks:
       - geoapi
     volumes:
-      - .:/app
+      - ../:/app
       - assets:/assets
     environment:
       - MAPILLARY_CLIENT_TOKEN=4866220476802272
@@ -85,25 +81,23 @@ services:
     command: "celery -A geoapi.celery_app beat -l info"
 
   api:
-    build:
-      context: .
-      dockerfile: ./Dockerfile
+    image: taccaci/geoapi:local
     networks:
       - geoapi
     volumes:
-      - .:/app
+      - ../:/app
       - assets:/assets
     ports:
       - 8000:8000
     dns:
       - 8.8.8.8
       - 8.8.4.4
     environment:
-      - MAPILLARY_CLIENT_TOKEN=4866220476802272
+      - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN}
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=development
       - ASSETS_BASE_DIR=/assets
-      - TENANT
+      - TENANT=${TENANT}
     stdin_open: true
     tty: true
     container_name: geoapi

docker-compose.test.yml → devops/docker-compose.test.yml

@@ -8,10 +8,10 @@ volumes:
 networks:
   geoapi:
     driver: bridge
-services:
 
+services:
   postgres_test:
-    image: mdillon/postgis:11-alpine
+    image: mdillon/postgis
     volumes:
       - pgdata:/var/lib/postgresql/data
     ports:
@@ -28,13 +28,9 @@ services:
       - POSTGRES_DB=test
 
   workers:
-    build:
-      context: .
-      dockerfile: ./Dockerfile.potree
-      cache_from:
-        - taccaci/geoapi-workers:latest
+    image: taccaci/geoapi-workers:local
     volumes:
-      - .:/app
+      - ..:/app
       - assets:/assets
     environment:
       - MAPILLARY_CLIENT_TOKEN=4866220476802272
@@ -52,13 +48,9 @@ services:
     hostname: geoapiworkers
 
   api:
-    build:
-      context: .
-      dockerfile: ./Dockerfile
-      cache_from:
-        - taccaci/geoapi:latest
+    image: taccaci/geoapi-workers:local
     volumes:
-      - .:/app
+      - ..:/app
       - assets:/assets
     ports:
       - 8000:8000

devops/geoapi-services/Makefile

@@ -0,0 +1,13 @@
+
+.PHONY: start
+start:
+	echo "starting (GEOAPI_TAG=${GEOAPI_TAG})"
+	docker compose -f docker-compose.yml up -d
+
+.PHONY: stop
+stop:
+	docker compose -f docker-compose.yml down
+
+.PHONY: pull
+pull:
+	docker compose pull api workers nginx rabbitmq