From 327a173fe2c3655dcbf8e9b33bb9b6339e99d26a Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 14 Sep 2023 10:08:34 -0500 Subject: [PATCH 01/49] Add initial/work-in-progress docker-compose --- devops/geoapi-services/docker-compose.yml | 96 +++++++++++++++++++++++ devops/geoapi-services/sample.env | 10 +++ 2 files changed, 106 insertions(+) create mode 100644 devops/geoapi-services/docker-compose.yml create mode 100644 devops/geoapi-services/sample.env diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml new file mode 100644 index 00000000..0febfadf --- /dev/null +++ b/devops/geoapi-services/docker-compose.yml @@ -0,0 +1,96 @@ +--- +version: "3" + +networks: + geoapi: + +services: + + rabbitmq: + image: rabbitmq + networks: + - geoapi + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} + - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_DEFAULT_VHOST} + + nginx: + image: nginx + ports: + - 8888:80 + volumes: + - /assets:/assets + - ./conf/nginx.conf:/etc/nginx/nginx.conf + networks: + - geoapi + + + workers: + build: + context: . + dockerfile: ./Dockerfile.potree + networks: + - geoapi + volumes: + - .:/app + - /assets:/assets + environment: + - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - FLASK_APP=/app/geoapi/app.py + - APP_ENV=development + - ASSETS_BASE_DIR=/assets + - TENANT + stdin_open: true + tty: true + container_name: geoapiworkers + hostname: geoapiworkers + command: "celery -A geoapi.celery_app worker -l info" + + celerybeat: + build: + context: . + dockerfile: ./Dockerfile.potree + networks: + - geoapi + volumes: + - .:/app + - assets:/assets + environment: + - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - FLASK_APP=/app/geoapi/app.py + - APP_ENV=development + - ASSETS_BASE_DIR=/assets + stdin_open: true + tty: true + container_name: geoapicelerybeat + hostname: geoapicelerybeat + command: "celery -A geoapi.celery_app beat -l info" + + api: + build: + context: . + dockerfile: ./Dockerfile + networks: + - geoapi + volumes: + - .:/app + - assets:/assets + ports: + - 8000:8000 + dns: + - 8.8.8.8 + - 8.8.4.4 + environment: + - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - FLASK_APP=/app/geoapi/app.py + - APP_ENV=development + - ASSETS_BASE_DIR=/assets + - TENANT + stdin_open: true + tty: true + container_name: geoapi + hostname: geoapi + command: "gunicorn -w 4 -b 0.0.0.0:8000 geoapi.app:app -k gevent --reload --timeout 300" + extra_hosts: + - "host.docker.internal:host-gateway" diff --git a/devops/geoapi-services/sample.env b/devops/geoapi-services/sample.env new file mode 100644 index 00000000..1086304c --- /dev/null +++ b/devops/geoapi-services/sample.env @@ -0,0 +1,10 @@ +RABBITMQ_DEFAULT_USER=dev +RABBITMQ_DEFAULT_PASS=dev +RABBITMQ_DEFAULT_VHOST=dev +APP_ENV=development # "production" for STAGING and PROD? +TENANT= + +# TODO where does nginx.conf come from / is store +# CHECK conf/nginx.conf and nginx.kube.conf and possibly hardocded in a yaml + +# TODO determine if we need to do TENANT or TENANT=${TENANT} \ No newline at end of file From 24fac06a4493f051b2b196d2231963d0447ecff9 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 14 Sep 2023 10:33:42 -0500 Subject: [PATCH 02/49] Add nginx conf --- conf/nginx.conf | 55 ----------------------- devops/geoapi-services/docker-compose.yml | 2 +- devops/geoapi-services/nginx.conf | 46 +++++++++++++++++++ 3 files changed, 47 insertions(+), 56 deletions(-) delete mode 100644 conf/nginx.conf create mode 100644 devops/geoapi-services/nginx.conf diff --git a/conf/nginx.conf b/conf/nginx.conf deleted file mode 100644 index e4069d17..00000000 --- a/conf/nginx.conf +++ /dev/null @@ -1,55 +0,0 @@ -events { - worker_connections 1024; # increase if you have lots of clients - accept_mutex off; # set to 'on' if nginx worker_processes > 1 - use epoll; -} -http { - - gzip on; - gzip_proxied any; - gzip_types *; - gzip_min_length 250; - - server { - include /etc/nginx/mime.types; - client_max_body_size 10g; - location / { - add_header "Access-Control-Allow-Origin" *; - - # Preflighted requests - if ($request_method = OPTIONS ) { - add_header "Access-Control-Allow-Origin" *; - add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; - add_header "Access-Control-Allow-Headers" "*"; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Length' 0; - return 204; - } - rewrite ^/api(.*) /$1 break; - proxy_pass http://geoapi:8000; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /assets { - max_ranges 0; - expires 30d; - add_header "Access-Control-Allow-Origin" *; - # Preflighted requests - if ($request_method = OPTIONS ) { - add_header "Access-Control-Allow-Origin" *; - add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; - add_header "Access-Control-Allow-Headers" "*"; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Length' 0; - return 204; - } - alias /assets/; - } - } -} diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index 0febfadf..6c6456c0 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -21,7 +21,7 @@ services: - 8888:80 volumes: - /assets:/assets - - ./conf/nginx.conf:/etc/nginx/nginx.conf + - ./nginx.conf:/etc/nginx/nginx.conf networks: - geoapi diff --git a/devops/geoapi-services/nginx.conf b/devops/geoapi-services/nginx.conf new file mode 100644 index 00000000..b715bf5a --- /dev/null +++ b/devops/geoapi-services/nginx.conf @@ -0,0 +1,46 @@ +server { + include /etc/nginx/mime.types; + client_max_body_size 25g; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_connect_timeout 20s; + + gzip on; + gzip_proxied any; + gzip_types *; + gzip_min_length 250; + + client_body_buffer_size 70m; + client_header_buffer_size 50k; + large_client_header_buffers 2 50k; + + location / { + add_header "Access-Control-Allow-Origin" *; + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } + rewrite ^/api(.*) /$1 break; + proxy_pass http://geoapi:5000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /assets { + max_ranges 0; + expires 30d; + add_header "Access-Control-Allow-Origin" *; + alias /assets/; + } +} \ No newline at end of file From 691075813ba83fd64487990130bbab31100022dc Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 14 Sep 2023 10:35:37 -0500 Subject: [PATCH 03/49] Move docker infrastructure to devops folder --- Makefile | 4 ++-- Dockerfile => devops/Dockerfile | 0 Dockerfile.potree => devops/Dockerfile.potree | 0 docker-compose.yml => devops/docker-compose.yml | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename Dockerfile => devops/Dockerfile (100%) rename Dockerfile.potree => devops/Dockerfile.potree (100%) rename docker-compose.yml => devops/docker-compose.yml (100%) diff --git a/Makefile b/Makefile index 97a0ac96..9c226e09 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,12 @@ GEOAPI_WORKERS=taccaci/geoapi-workers .PHONY: geoapi geoapi: - docker build -t $(GEOAPI_IMAGE):$(TAG) . + docker build -t $(GEOAPI_IMAGE):$(TAG) -f devops/Dockerfile . docker tag $(GEOAPI_IMAGE):$(TAG) $(GEOAPI_IMAGE):latest .PHONY: workers workers: - docker build -t $(GEOAPI_WORKERS):$(TAG) -f Dockerfile.potree . + docker build -t $(GEOAPI_WORKERS):$(TAG) -f devops/Dockerfile.potree . docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):latest .PHONY: deploy-geoapi diff --git a/Dockerfile b/devops/Dockerfile similarity index 100% rename from Dockerfile rename to devops/Dockerfile diff --git a/Dockerfile.potree b/devops/Dockerfile.potree similarity index 100% rename from Dockerfile.potree rename to devops/Dockerfile.potree diff --git a/docker-compose.yml b/devops/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to devops/docker-compose.yml From 443c96a64348cd34756ab6a251a1261818f742ba Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 14 Sep 2023 10:57:47 -0500 Subject: [PATCH 04/49] Move remaining docker infrastructure to devops folder --- Makefile | 8 ++++++++ README.md | 2 +- ...{docker-compose.yml => docker-compose.local.yml} | 8 ++++---- .../misc}/potree/page_template/nsf_logo.png | Bin .../misc}/potree/page_template/viewer_template.html | 0 poetry.lock => devops/poetry.lock | 0 pyproject.toml => devops/pyproject.toml | 0 7 files changed, 13 insertions(+), 5 deletions(-) rename devops/{docker-compose.yml => docker-compose.local.yml} (93%) rename {misc => devops/misc}/potree/page_template/nsf_logo.png (100%) rename {misc => devops/misc}/potree/page_template/viewer_template.html (100%) rename poetry.lock => devops/poetry.lock (100%) rename pyproject.toml => devops/pyproject.toml (100%) diff --git a/Makefile b/Makefile index 9c226e09..dd21804c 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,14 @@ TAG := $(shell git log --format=%h -1) GEOAPI_IMAGE=taccaci/geoapi GEOAPI_WORKERS=taccaci/geoapi-workers +.PHONY: start +start: + docker-compose -f devops/docker-compose.local.yml --env-file .env up + +.PHONY: stop +stop: + docker-compose -f devops/docker-compose.local.yml --env-file .env down + .PHONY: geoapi geoapi: docker build -t $(GEOAPI_IMAGE):$(TAG) -f devops/Dockerfile . diff --git a/README.md b/README.md index 0985c4ee..e5d74357 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ information. An .env file for developers can be found on [UT Stache](https://sta The API is built with flask and flask-restplus. It is running in its own container under gunicorn on port 8000 -`docker-compose up` +`make start` ###### Initialize the database diff --git a/devops/docker-compose.yml b/devops/docker-compose.local.yml similarity index 93% rename from devops/docker-compose.yml rename to devops/docker-compose.local.yml index d512233e..e78ec244 100644 --- a/devops/docker-compose.yml +++ b/devops/docker-compose.local.yml @@ -53,11 +53,11 @@ services: - .:/app - assets:/assets environment: - - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets - - TENANT + - TENANT=${TENANT} stdin_open: true tty: true container_name: geoapiworkers @@ -99,11 +99,11 @@ services: - 8.8.8.8 - 8.8.4.4 environment: - - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets - - TENANT + - TENANT=${TENANT} stdin_open: true tty: true container_name: geoapi diff --git a/misc/potree/page_template/nsf_logo.png b/devops/misc/potree/page_template/nsf_logo.png similarity index 100% rename from misc/potree/page_template/nsf_logo.png rename to devops/misc/potree/page_template/nsf_logo.png diff --git a/misc/potree/page_template/viewer_template.html b/devops/misc/potree/page_template/viewer_template.html similarity index 100% rename from misc/potree/page_template/viewer_template.html rename to devops/misc/potree/page_template/viewer_template.html diff --git a/poetry.lock b/devops/poetry.lock similarity index 100% rename from poetry.lock rename to devops/poetry.lock diff --git a/pyproject.toml b/devops/pyproject.toml similarity index 100% rename from pyproject.toml rename to devops/pyproject.toml From b15b0fc9cdcc1508ce1f129a926a0cdfde1266c8 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 14 Sep 2023 13:59:23 -0500 Subject: [PATCH 05/49] Fix local development's docker-compose --- conf/nginx.kube.conf | 18 ----------- devops/docker-compose.local.yml | 6 ++-- devops/local_conf/nginx.conf | 55 +++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 21 deletions(-) delete mode 100644 conf/nginx.kube.conf create mode 100644 devops/local_conf/nginx.conf diff --git a/conf/nginx.kube.conf b/conf/nginx.kube.conf deleted file mode 100644 index 79abe44f..00000000 --- a/conf/nginx.kube.conf +++ /dev/null @@ -1,18 +0,0 @@ -events { - worker_connections 1024; # increase if you have lots of clients - accept_mutex off; # set to 'on' if nginx worker_processes > 1 - use epoll; -} -http { - server { - include /etc/nginx/mime.types; - client_max_body_size 10g; - - location /assets { - max_ranges 0; - expires 30d; - add_header "Access-Control-Allow-Origin" *; - alias /assets/; - } - } -} \ No newline at end of file diff --git a/devops/docker-compose.local.yml b/devops/docker-compose.local.yml index e78ec244..ec73432a 100644 --- a/devops/docker-compose.local.yml +++ b/devops/docker-compose.local.yml @@ -38,7 +38,7 @@ services: - 8888:80 volumes: - assets:/assets - - ./conf/nginx.conf:/etc/nginx/nginx.conf + - ./local_conf/nginx.conf:/etc/nginx/nginx.conf networks: - geoapi @@ -50,7 +50,7 @@ services: networks: - geoapi volumes: - - .:/app + - ../:/app - assets:/assets environment: - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} @@ -91,7 +91,7 @@ services: networks: - geoapi volumes: - - .:/app + - ../:/app - assets:/assets ports: - 8000:8000 diff --git a/devops/local_conf/nginx.conf b/devops/local_conf/nginx.conf new file mode 100644 index 00000000..e4069d17 --- /dev/null +++ b/devops/local_conf/nginx.conf @@ -0,0 +1,55 @@ +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 + use epoll; +} +http { + + gzip on; + gzip_proxied any; + gzip_types *; + gzip_min_length 250; + + server { + include /etc/nginx/mime.types; + client_max_body_size 10g; + location / { + add_header "Access-Control-Allow-Origin" *; + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } + rewrite ^/api(.*) /$1 break; + proxy_pass http://geoapi:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /assets { + max_ranges 0; + expires 30d; + add_header "Access-Control-Allow-Origin" *; + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } + alias /assets/; + } + } +} From 49774e1eceb081e2c9014862fa709b2d1ccffed7 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 14 Sep 2023 13:59:44 -0500 Subject: [PATCH 06/49] Add additional improvements to docker-compose for geoapi-services vms --- devops/geoapi-services/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index 6c6456c0..e1346876 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -29,14 +29,14 @@ services: workers: build: context: . - dockerfile: ./Dockerfile.potree + dockerfile: ../Dockerfile.potree networks: - geoapi volumes: - .:/app - /assets:/assets environment: - - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets @@ -50,7 +50,7 @@ services: celerybeat: build: context: . - dockerfile: ./Dockerfile.potree + dockerfile: ../Dockerfile.potree networks: - geoapi volumes: @@ -70,7 +70,7 @@ services: api: build: context: . - dockerfile: ./Dockerfile + dockerfile: ../Dockerfile networks: - geoapi volumes: @@ -86,7 +86,7 @@ services: - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets - - TENANT + - TENANT=${TENANT} stdin_open: true tty: true container_name: geoapi From 466400ea4f94f989fa8742c7fcb33c39ca570b96 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Mon, 18 Sep 2023 16:57:47 -0500 Subject: [PATCH 07/49] Add some variables --- devops/geoapi-services/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index e1346876..f2538f68 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -40,7 +40,7 @@ services: - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets - - TENANT + - TENANT=${TENANT} stdin_open: true tty: true container_name: geoapiworkers @@ -57,7 +57,7 @@ services: - .:/app - assets:/assets environment: - - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets @@ -82,7 +82,7 @@ services: - 8.8.8.8 - 8.8.4.4 environment: - - MAPILLARY_CLIENT_TOKEN=4866220476802272 + - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - APP_ENV=development - ASSETS_BASE_DIR=/assets From 37486df121d493c9b8014b337fa4c5324d61a2f4 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 11:42:06 -0500 Subject: [PATCH 08/49] Use one env variable for mapillary --- devops/geoapi-services/sample.env | 8 ++------ geoapi/utils/streetview.py | 2 +- kube/Makefile | 2 +- kube/geoapi.yaml | 1 - kube/prod.env | 3 +-- kube/staging.env | 3 +-- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/devops/geoapi-services/sample.env b/devops/geoapi-services/sample.env index 1086304c..334de99b 100644 --- a/devops/geoapi-services/sample.env +++ b/devops/geoapi-services/sample.env @@ -2,9 +2,5 @@ RABBITMQ_DEFAULT_USER=dev RABBITMQ_DEFAULT_PASS=dev RABBITMQ_DEFAULT_VHOST=dev APP_ENV=development # "production" for STAGING and PROD? -TENANT= - -# TODO where does nginx.conf come from / is store -# CHECK conf/nginx.conf and nginx.kube.conf and possibly hardocded in a yaml - -# TODO determine if we need to do TENANT or TENANT=${TENANT} \ No newline at end of file +TENANT="{'DESIGNSAFE': {'service_account_token': 'ABCD1234'}}" +MAPILLARY_CLIENT_TOKEN="ABC|1234564675757|abcd1234abcd12345" diff --git a/geoapi/utils/streetview.py b/geoapi/utils/streetview.py index 648c848d..8eb197f9 100644 --- a/geoapi/utils/streetview.py +++ b/geoapi/utils/streetview.py @@ -100,7 +100,7 @@ def authenticate(userId: int, jwt: str, service_user: str): subprocess.run(command, check=True, env={ - 'MAPILLARY_CLIENT_TOKEN': settings.MAPILLARY_CLIENT_ID, + 'MAPILLARY_CLIENT_TOKEN': settings.MAPILLARY_CLIENT_TOKEN, 'MAPILLARY_CONFIG_PATH': MapillaryUtils.get_auth_file(userId) }) except subprocess.CalledProcessError as e: diff --git a/kube/Makefile b/kube/Makefile index 1c1d55a1..c723abe0 100644 --- a/kube/Makefile +++ b/kube/Makefile @@ -27,7 +27,7 @@ checkfortag: .PHONY: create create: checkforcontext checkfortag @echo "Creating geoapi deployment/services/migration-job in '$(KUBE_CONTEXT)' context" - source ${ENV_FILE} && cat geoapi.yaml | envsubst '$${GEOAPI_TAG} $${NODEPORT} $${NSFSHARE_CLUSTER_IP} $${MAPILLARY_CLIENT_TOKEN} $${MAPILLARY_CLIENT_ID}' | kubectl apply --context $(KUBE_CONTEXT) -f - + source ${ENV_FILE} && cat geoapi.yaml | envsubst '$${GEOAPI_TAG} $${NODEPORT} $${NSFSHARE_CLUSTER_IP} $${MAPILLARY_CLIENT_TOKEN}' | kubectl apply --context $(KUBE_CONTEXT) -f - .PHONY: delete delete: checkforcontext diff --git a/kube/geoapi.yaml b/kube/geoapi.yaml index a60d885f..fd4725c5 100644 --- a/kube/geoapi.yaml +++ b/kube/geoapi.yaml @@ -65,7 +65,6 @@ data: ASSETS_BASE_DIR: /assets/ JWT_SECRET_KEY: your-256-bit-secret MAPILLARY_CLIENT_TOKEN: '${MAPILLARY_CLIENT_TOKEN}' - MAPILLARY_CLIENT_ID: '${MAPILLARY_CLIENT_ID}' FLASK_APP: /app/geoapi/app.py APP_ENV: production RABBITMQ_HOSTNAME: geoapi-rabbitmq diff --git a/kube/prod.env b/kube/prod.env index cd859961..6f60a9ad 100644 --- a/kube/prod.env +++ b/kube/prod.env @@ -1,4 +1,3 @@ export NODEPORT=31474 export NSFSHARE_CLUSTER_IP=10.102.87.187 -export MAPILLARY_CLIENT_TOKEN='5156692464392931' -export MAPILLARY_CLIENT_ID='MLY|5156692464392931|4f1118aa1b06f051a44217cb56bedf79' +export MAPILLARY_CLIENT_TOKEN='MLY|5156692464392931|4f1118aa1b06f051a44217cb56bedf79' diff --git a/kube/staging.env b/kube/staging.env index 97c12157..43e97635 100644 --- a/kube/staging.env +++ b/kube/staging.env @@ -1,4 +1,3 @@ export NODEPORT=31938 export NSFSHARE_CLUSTER_IP=10.106.172.196 -export MAPILLARY_CLIENT_TOKEN='4936281379826603' -export MAPILLARY_CLIENT_ID='MLY|4936281379826603|f8c4732d3c9d96582b86158feb1c1a7a' +export MAPILLARY_CLIENT_TOKEN='MLY|4936281379826603|f8c4732d3c9d96582b86158feb1c1a7a' From ee47a45c37d6d83e6958ff0467a94c8c9d42ef32 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 19:58:07 -0500 Subject: [PATCH 09/49] Fix dockerfiles --- .github/workflows/main.yml | 2 +- devops/Dockerfile | 2 +- devops/Dockerfile.potree | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03410e88..8c81765b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,7 +86,7 @@ jobs: - name: Build worker image run: | docker pull taccaci/geoapi-workers:latest - docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f Dockerfile.potree . + docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.potree . - name: Run worker test run: | docker run --network="host" -e APP_ENV='testing' -e DB_HOST='localhost' taccaci/geoapi-workers:latest pytest -m "worker" diff --git a/devops/Dockerfile b/devops/Dockerfile index d77b4658..342c9da0 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -12,7 +12,7 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY pyproject.toml poetry.lock ./ +COPY devops/pyproject.toml devops/poetry.lock ./ RUN poetry install RUN mkdir /app diff --git a/devops/Dockerfile.potree b/devops/Dockerfile.potree index f9a0fa01..70ec5c0f 100644 --- a/devops/Dockerfile.potree +++ b/devops/Dockerfile.potree @@ -21,7 +21,7 @@ RUN git clone -b develop https://github.com/potree/PotreeConverter.git && cd Pot mkdir build && cd build && \ cmake -DCMAKE_BUILD_TYPE=Release -DLASZIP_INCLUDE_DIRS=/opt/LAStools/LASzip/dll/ -DLASZIP_LIBRARY=/usr/local/lib/liblaszip.so .. && \ make && make install && cp -r /opt/PotreeConverter/PotreeConverter/resources /resources -ADD misc/potree/page_template /resources/page_template +ADD devops/misc/potree/page_template /resources/page_template RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 @@ -32,7 +32,7 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY pyproject.toml poetry.lock ./ +COPY devops/pyproject.toml devops/poetry.lock ./ RUN poetry install ENV PYTHONPATH "${PYTHONPATH}:/app" From aa112f80f4b06e8ed2a6dbedcc1988fda88bcaa1 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 20:11:46 -0500 Subject: [PATCH 10/49] Add secret.env to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4c7a74f4..24eaba21 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ # Secret env file .env +*secret.env # C extensions *.so From 33b67993be6b9b22ce939a30f501dd8d9d0879d5 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 20:12:11 -0500 Subject: [PATCH 11/49] Update docker-compose test yaml --- docker-compose.test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 48ab981d..bf4a246c 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -11,7 +11,7 @@ networks: services: postgres_test: - image: mdillon/postgis:11-alpine + image: mdillon/postgis volumes: - pgdata:/var/lib/postgresql/data ports: @@ -30,7 +30,7 @@ services: workers: build: context: . - dockerfile: ./Dockerfile.potree + dockerfile: ./devops/Dockerfile.potree cache_from: - taccaci/geoapi-workers:latest volumes: @@ -54,7 +54,7 @@ services: api: build: context: . - dockerfile: ./Dockerfile + dockerfile: ./devops/Dockerfile cache_from: - taccaci/geoapi:latest volumes: From 4b5a55781516c612edea89e1577aaefbd5b3a082 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 20:17:18 -0500 Subject: [PATCH 12/49] Fix installing of python packages on github actions workflow --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c81765b..28739eff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,9 @@ jobs: restore-keys: | ${{ runner.os }}-poetry- - name: Install Python Packages - run: poetry install + run: | + cd devops + poetry install - name: Run server-side unit tests run: poetry run pytest Workers_Unit_Tests: From d3ff8b3a2163b79963b0e6f59f2915e43f09fa40 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 20:19:48 -0500 Subject: [PATCH 13/49] Change to devops/ before running test on CI workflow --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28739eff..faa861c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,7 +62,9 @@ jobs: cd devops poetry install - name: Run server-side unit tests - run: poetry run pytest + run: | + cd devops + poetry run pytest Workers_Unit_Tests: runs-on: ubuntu-latest services: From cf069c6df3ce68b7feb39c14a74c77c93bb3ed6b Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 19 Sep 2023 20:24:57 -0500 Subject: [PATCH 14/49] Simplify install/pytest step in CI workflow --- .github/workflows/main.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index faa861c3..12ab0e09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,13 +58,9 @@ jobs: restore-keys: | ${{ runner.os }}-poetry- - name: Install Python Packages - run: | - cd devops - poetry install + run: poetry install --path devops - name: Run server-side unit tests - run: | - cd devops - poetry run pytest + run: poetry run pytest Workers_Unit_Tests: runs-on: ubuntu-latest services: From d397f14c0093a09e7578568916474bbd3701870b Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 21 Sep 2023 15:27:50 -0500 Subject: [PATCH 15/49] Refactor location of dockerfiles and images used --- Makefile | 2 + README.md | 9 +++-- devops/Dockerfile | 4 +- devops/Dockerfile.potree | 6 +-- devops/docker-compose.local.yml | 14 ++----- .../docker-compose.test.yml | 18 +++------ devops/geoapi-services/docker-compose.yml | 40 +++++-------------- devops/geoapi-services/sample.env | 5 +++ devops/poetry.lock => poetry.lock | 0 devops/pyproject.toml => pyproject.toml | 0 10 files changed, 38 insertions(+), 60 deletions(-) rename docker-compose.test.yml => devops/docker-compose.test.yml (82%) rename devops/poetry.lock => poetry.lock (100%) rename devops/pyproject.toml => pyproject.toml (100%) diff --git a/Makefile b/Makefile index dd21804c..06758406 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,13 @@ stop: geoapi: docker build -t $(GEOAPI_IMAGE):$(TAG) -f devops/Dockerfile . docker tag $(GEOAPI_IMAGE):$(TAG) $(GEOAPI_IMAGE):latest + docker tag $(GEOAPI_IMAGE):$(TAG) $(GEOAPI_IMAGE):local .PHONY: workers workers: docker build -t $(GEOAPI_WORKERS):$(TAG) -f devops/Dockerfile.potree . docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):latest + docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):local .PHONY: deploy-geoapi deploy-geoapi: diff --git a/README.md b/README.md index e5d74357..77376659 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ information. An .env file for developers can be found on [UT Stache](https://sta The API is built with flask and flask-restplus. It is running in its own container under gunicorn on port 8000 +`make geoapi` +`make workers` `make start` ###### Initialize the database @@ -87,17 +89,18 @@ alembic revision --autogenerate Run route/service tests on the `api` container ``` -docker-compose -f docker-compose.test.yml -p geoapi_test run api pytest +docker-compose -f devops/docker-compose.test.yml -p geoapi_test run api pytest ``` Run worker-related tasks on the `workers` container ``` -docker-compose -f docker-compose.test.yml -p geoapi_test run workers pytest -m "worker" +docker-compose -f devops/docker-compose.test.yml -p geoapi_test run workers pytest -m "worker" ``` Note that images need to be rebuilt before running tests if they have been updated (e.g. packages): ``` -docker-compose -f docker-compose.test.yml build +make workers +make geoapi ``` ## Kubernetes (Production/Staging) diff --git a/devops/Dockerfile b/devops/Dockerfile index 342c9da0..81c8e1cd 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -12,10 +12,10 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY devops/pyproject.toml devops/poetry.lock ./ +COPY ../pyproject.toml ../poetry.lock ./ RUN poetry install RUN mkdir /app -COPY ./geoapi /app/geoapi +COPY ../geoapi /app/geoapi ENV PYTHONPATH "${PYTHONPATH}:/app" WORKDIR /app/geoapi diff --git a/devops/Dockerfile.potree b/devops/Dockerfile.potree index 70ec5c0f..135cf380 100644 --- a/devops/Dockerfile.potree +++ b/devops/Dockerfile.potree @@ -32,16 +32,16 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY devops/pyproject.toml devops/poetry.lock ./ +COPY ../pyproject.toml ../poetry.lock ./ RUN poetry install ENV PYTHONPATH "${PYTHONPATH}:/app" -# isntall geos into condo the base pdal image is using +# install geos into condo the base pdal image is using RUN conda install geos -y -n base WORKDIR / RUN mkdir app -COPY ./geoapi /app/geoapi +COPY ../geoapi /app/geoapi WORKDIR /app/geoapi diff --git a/devops/docker-compose.local.yml b/devops/docker-compose.local.yml index ec73432a..a1986876 100644 --- a/devops/docker-compose.local.yml +++ b/devops/docker-compose.local.yml @@ -44,9 +44,7 @@ services: workers: - build: - context: . - dockerfile: ./Dockerfile.potree + image: taccaci/geoapi-workers:local networks: - geoapi volumes: @@ -65,13 +63,11 @@ services: command: "celery -A geoapi.celery_app worker -l info" celerybeat: - build: - context: . - dockerfile: ./Dockerfile.potree + image: taccaci/geoapi-workers:local networks: - geoapi volumes: - - .:/app + - ../:/app - assets:/assets environment: - MAPILLARY_CLIENT_TOKEN=4866220476802272 @@ -85,9 +81,7 @@ services: command: "celery -A geoapi.celery_app beat -l info" api: - build: - context: . - dockerfile: ./Dockerfile + image: taccaci/geoapi:local networks: - geoapi volumes: diff --git a/docker-compose.test.yml b/devops/docker-compose.test.yml similarity index 82% rename from docker-compose.test.yml rename to devops/docker-compose.test.yml index bf4a246c..d29b88bf 100644 --- a/docker-compose.test.yml +++ b/devops/docker-compose.test.yml @@ -8,8 +8,8 @@ volumes: networks: geoapi: driver: bridge -services: +services: postgres_test: image: mdillon/postgis volumes: @@ -28,13 +28,9 @@ services: - POSTGRES_DB=test workers: - build: - context: . - dockerfile: ./devops/Dockerfile.potree - cache_from: - - taccaci/geoapi-workers:latest + image: taccaci/geoapi-workers:local volumes: - - .:/app + - ..:/app - assets:/assets environment: - MAPILLARY_CLIENT_TOKEN=4866220476802272 @@ -52,13 +48,9 @@ services: hostname: geoapiworkers api: - build: - context: . - dockerfile: ./devops/Dockerfile - cache_from: - - taccaci/geoapi:latest + image: taccaci/geoapi-workers:local volumes: - - .:/app + - ..:/app - assets:/assets ports: - 8000:8000 diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index f2538f68..cd60a7dc 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -5,15 +5,11 @@ networks: geoapi: services: - rabbitmq: - image: rabbitmq + image: rabbitmq:3.8.18 networks: - geoapi - environment: - - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} - - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} - - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_DEFAULT_VHOST} + env_file: ./secrets.env nginx: image: nginx @@ -25,22 +21,16 @@ services: networks: - geoapi - workers: - build: - context: . - dockerfile: ../Dockerfile.potree + image: taccaci/geoapi-workers:${GEOAPI_TAG} networks: - geoapi volumes: - - .:/app - /assets:/assets + env_file: ./secrets.env environment: - - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - - APP_ENV=development - ASSETS_BASE_DIR=/assets - - TENANT=${TENANT} stdin_open: true tty: true container_name: geoapiworkers @@ -48,18 +38,14 @@ services: command: "celery -A geoapi.celery_app worker -l info" celerybeat: - build: - context: . - dockerfile: ../Dockerfile.potree + image: taccaci/geoapi-workers:${GEOAPI_TAG} networks: - geoapi volumes: - - .:/app - - assets:/assets + - /assets:/assets + env_file: ./secrets.env environment: - - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - - APP_ENV=development - ASSETS_BASE_DIR=/assets stdin_open: true tty: true @@ -68,25 +54,21 @@ services: command: "celery -A geoapi.celery_app beat -l info" api: - build: - context: . - dockerfile: ../Dockerfile + image: taccaci/geoapi:${GEOAPI_TAG} networks: - geoapi volumes: - - .:/app - - assets:/assets + - /assets:/assets ports: - 8000:8000 dns: - 8.8.8.8 - 8.8.4.4 + env_file: ./secrets.env environment: - - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN} - FLASK_APP=/app/geoapi/app.py - - APP_ENV=development - ASSETS_BASE_DIR=/assets - - TENANT=${TENANT} + stdin_open: true tty: true container_name: geoapi diff --git a/devops/geoapi-services/sample.env b/devops/geoapi-services/sample.env index 334de99b..ed76d35a 100644 --- a/devops/geoapi-services/sample.env +++ b/devops/geoapi-services/sample.env @@ -1,6 +1,11 @@ +DB_USERNAME=dev +DB_NAME=dev +DB_PASSWD=dev +DB_HOST=${DB_HOST} RABBITMQ_DEFAULT_USER=dev RABBITMQ_DEFAULT_PASS=dev RABBITMQ_DEFAULT_VHOST=dev +RABBITMQ_HOSTNAME=rabbitmq APP_ENV=development # "production" for STAGING and PROD? TENANT="{'DESIGNSAFE': {'service_account_token': 'ABCD1234'}}" MAPILLARY_CLIENT_TOKEN="ABC|1234564675757|abcd1234abcd12345" diff --git a/devops/poetry.lock b/poetry.lock similarity index 100% rename from devops/poetry.lock rename to poetry.lock diff --git a/devops/pyproject.toml b/pyproject.toml similarity index 100% rename from devops/pyproject.toml rename to pyproject.toml From f6ba45eb2293f4ec1c3b5a04a1bf8198c16194a9 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 21 Sep 2023 15:28:09 -0500 Subject: [PATCH 16/49] Fix nginx conf for deployment --- devops/geoapi-services/nginx.conf | 86 +++++++++++++++++-------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/devops/geoapi-services/nginx.conf b/devops/geoapi-services/nginx.conf index b715bf5a..693e5e10 100644 --- a/devops/geoapi-services/nginx.conf +++ b/devops/geoapi-services/nginx.conf @@ -1,46 +1,54 @@ -server { - include /etc/nginx/mime.types; - client_max_body_size 25g; - proxy_read_timeout 600s; - proxy_send_timeout 600s; - proxy_connect_timeout 20s; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 + use epoll; +} - gzip on; - gzip_proxied any; - gzip_types *; - gzip_min_length 250; +http { + server { + include /etc/nginx/mime.types; + client_max_body_size 25g; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_connect_timeout 20s; - client_body_buffer_size 70m; - client_header_buffer_size 50k; - large_client_header_buffers 2 50k; + gzip on; + gzip_proxied any; + gzip_types *; + gzip_min_length 250; - location / { - add_header "Access-Control-Allow-Origin" *; + client_body_buffer_size 70m; + client_header_buffer_size 50k; + large_client_header_buffers 2 50k; - # Preflighted requests - if ($request_method = OPTIONS ) { - add_header "Access-Control-Allow-Origin" *; - add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; - add_header "Access-Control-Allow-Headers" "*"; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Length' 0; - return 204; + location / { + add_header "Access-Control-Allow-Origin" *; + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } + rewrite ^/api(.*) /$1 break; + proxy_pass http://geoapi:5000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - rewrite ^/api(.*) /$1 break; - proxy_pass http://geoapi:5000; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - location /assets { - max_ranges 0; - expires 30d; - add_header "Access-Control-Allow-Origin" *; - alias /assets/; - } + location /assets { + max_ranges 0; + expires 30d; + add_header "Access-Control-Allow-Origin" *; + alias /assets/; + } + } } \ No newline at end of file From aca6a94a34f1fbb26982089a5114b7adaa8ecec7 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 10:09:09 -0500 Subject: [PATCH 17/49] Change secrets location to /opt/portal/conf/secret.env --- devops/geoapi-services/{sample.env => secrets.sample.env} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename devops/geoapi-services/{sample.env => secrets.sample.env} (100%) diff --git a/devops/geoapi-services/sample.env b/devops/geoapi-services/secrets.sample.env similarity index 100% rename from devops/geoapi-services/sample.env rename to devops/geoapi-services/secrets.sample.env From 0d05626aa6286485389867f882d41b8db21a9ac3 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 10:39:46 -0500 Subject: [PATCH 18/49] Use absolute path for secrets.env --- devops/geoapi-services/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index cd60a7dc..cd1f7de6 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -9,7 +9,7 @@ services: image: rabbitmq:3.8.18 networks: - geoapi - env_file: ./secrets.env + env_file: /opt/portal/conf/secrets.env nginx: image: nginx @@ -27,7 +27,7 @@ services: - geoapi volumes: - /assets:/assets - env_file: ./secrets.env + env_file: /opt/portal/conf/secrets.env environment: - FLASK_APP=/app/geoapi/app.py - ASSETS_BASE_DIR=/assets @@ -43,7 +43,7 @@ services: - geoapi volumes: - /assets:/assets - env_file: ./secrets.env + env_file: /opt/portal/conf/secrets.env environment: - FLASK_APP=/app/geoapi/app.py - ASSETS_BASE_DIR=/assets @@ -64,7 +64,7 @@ services: dns: - 8.8.8.8 - 8.8.4.4 - env_file: ./secrets.env + env_file: /opt/portal/conf/secrets.env environment: - FLASK_APP=/app/geoapi/app.py - ASSETS_BASE_DIR=/assets From 00669c49649daaf3c90b1854572a434a3d414a17 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 11:12:58 -0500 Subject: [PATCH 19/49] Add makefile for geoapi-services --- devops/geoapi-services/Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 devops/geoapi-services/Makefile diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile new file mode 100644 index 00000000..bc0ac0e2 --- /dev/null +++ b/devops/geoapi-services/Makefile @@ -0,0 +1,11 @@ +ENV_FILE ?= /opt/portal/secrets.env +# get GEOAPI_TAG and HAZMAPPER_TAG to be used by docker-compose.yml +include $(ENV_FILE) + +.PHONY: start +start: + docker-compose -f docker-compose.yml up + +.PHONY: stop +stop: + docker-compose -f docker-compose.yml down From e8d3447528495447992febfa96a322477c94444f Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 12:00:50 -0500 Subject: [PATCH 20/49] Refactor dockerfile --- .github/workflows/main.yml | 2 +- devops/Dockerfile | 8 ++++++-- devops/Dockerfile.potree | 4 +++- poetry.lock => devops/poetry.lock | 0 pyproject.toml => devops/pyproject.toml | 0 5 files changed, 10 insertions(+), 4 deletions(-) rename poetry.lock => devops/poetry.lock (100%) rename pyproject.toml => devops/pyproject.toml (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 12ab0e09..8c81765b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,7 @@ jobs: restore-keys: | ${{ runner.os }}-poetry- - name: Install Python Packages - run: poetry install --path devops + run: poetry install - name: Run server-side unit tests run: poetry run pytest Workers_Unit_Tests: diff --git a/devops/Dockerfile b/devops/Dockerfile index 81c8e1cd..9762d268 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -12,10 +12,14 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY ../pyproject.toml ../poetry.lock ./ -RUN poetry install RUN mkdir /app +WORKDIR /app + +COPY poetry.lock ./ +COPY pyproject.toml ./ +RUN poetry install + COPY ../geoapi /app/geoapi ENV PYTHONPATH "${PYTHONPATH}:/app" WORKDIR /app/geoapi diff --git a/devops/Dockerfile.potree b/devops/Dockerfile.potree index 135cf380..4e40d400 100644 --- a/devops/Dockerfile.potree +++ b/devops/Dockerfile.potree @@ -32,7 +32,9 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY ../pyproject.toml ../poetry.lock ./ +COPY pyproject.toml ./ +COPY poetry.lock ./ + RUN poetry install ENV PYTHONPATH "${PYTHONPATH}:/app" diff --git a/poetry.lock b/devops/poetry.lock similarity index 100% rename from poetry.lock rename to devops/poetry.lock diff --git a/pyproject.toml b/devops/pyproject.toml similarity index 100% rename from pyproject.toml rename to devops/pyproject.toml From 16e34d78a6c3a3b1245d7c1a59814e6103a5bf03 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 13:17:20 -0500 Subject: [PATCH 21/49] Correct path to poetry.lock and pyproject.toml --- devops/Dockerfile | 3 +-- devops/Dockerfile.potree | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/devops/Dockerfile b/devops/Dockerfile index 9762d268..07cefe05 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -16,8 +16,7 @@ RUN poetry config virtualenvs.create false RUN mkdir /app WORKDIR /app -COPY poetry.lock ./ -COPY pyproject.toml ./ +COPY devops/poetry.lock devops/pyproject.toml ./ RUN poetry install COPY ../geoapi /app/geoapi diff --git a/devops/Dockerfile.potree b/devops/Dockerfile.potree index 4e40d400..6e035938 100644 --- a/devops/Dockerfile.potree +++ b/devops/Dockerfile.potree @@ -32,8 +32,7 @@ ENV POETRY_HOME=/opt/poetry ENV PATH="$POETRY_HOME/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - RUN poetry config virtualenvs.create false -COPY pyproject.toml ./ -COPY poetry.lock ./ +COPY devops/pyproject.toml devops/poetry.lock ./ RUN poetry install From 10ccc52d7356635e693049ec9492ac0619553b5d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 13:30:15 -0500 Subject: [PATCH 22/49] Fix paths in dockerfile --- devops/Dockerfile | 2 +- devops/Dockerfile.potree | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devops/Dockerfile b/devops/Dockerfile index 07cefe05..be9fb290 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -19,6 +19,6 @@ WORKDIR /app COPY devops/poetry.lock devops/pyproject.toml ./ RUN poetry install -COPY ../geoapi /app/geoapi +COPY geoapi /app/geoapi ENV PYTHONPATH "${PYTHONPATH}:/app" WORKDIR /app/geoapi diff --git a/devops/Dockerfile.potree b/devops/Dockerfile.potree index 6e035938..c9c88c42 100644 --- a/devops/Dockerfile.potree +++ b/devops/Dockerfile.potree @@ -44,5 +44,5 @@ RUN conda install geos -y -n base WORKDIR / RUN mkdir app -COPY ../geoapi /app/geoapi +COPY geoapi /app/geoapi WORKDIR /app/geoapi From 107b95087d4df8e0f110b38e526f7d1f198c2886 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 13:36:41 -0500 Subject: [PATCH 23/49] Rename Dockerfile.potree to Dockerfile.worker --- .travis.yml | 34 ------------------- .../{Dockerfile.potree => Dockerfile.worker} | 0 2 files changed, 34 deletions(-) delete mode 100644 .travis.yml rename devops/{Dockerfile.potree => Dockerfile.worker} (100%) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f27f4efb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -language: generic -sudo: required -env: - global: - - GIT_HASH="$(git log --format=%h -1)" -services: - - docker - -jobs: - include: - - name: "geoapi: unit tests / container" - before_script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -t taccaci/geoapi:latest . - - sudo mkdir /assets - - sudo service mysql stop - - sudo service postgresql stop - - docker network create -d bridge test - - docker run -d --net=test -p 5432:5432 -e POSTGRES_DB=test -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev --hostname=postgres --name=postgres mdillon/postgis - - sleep 5 - script: - - docker run --net=test -e APP_ENV=testing taccaci/geoapi:latest pytest --cov=geoapi -m "not worker" - - name: "geoapi-workers: unit tests / container" - before_script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -t taccaci/geoapi-workers:latest -f Dockerfile.potree . - - sudo mkdir /assets - - sudo service mysql stop - - sudo service postgresql stop - - docker network create -d bridge test - - docker run -d --net=test -p 5432:5432 -e POSTGRES_DB=test -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev --hostname=postgres --name=postgres mdillon/postgis - - sleep 5 - script: - - docker run --net=test -e APP_ENV=testing taccaci/geoapi-workers:latest pytest --cov=geoapi -m "worker" diff --git a/devops/Dockerfile.potree b/devops/Dockerfile.worker similarity index 100% rename from devops/Dockerfile.potree rename to devops/Dockerfile.worker From 0ba35b9c2130bf681057d7e422563eb45a121200 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 22 Sep 2023 13:43:19 -0500 Subject: [PATCH 24/49] Rename Dockerfile.potree to Dockerfile.worker again --- .github/workflows/main.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c81765b..652a06e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,7 +86,7 @@ jobs: - name: Build worker image run: | docker pull taccaci/geoapi-workers:latest - docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.potree . + docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.worker . - name: Run worker test run: | docker run --network="host" -e APP_ENV='testing' -e DB_HOST='localhost' taccaci/geoapi-workers:latest pytest -m "worker" diff --git a/Makefile b/Makefile index 06758406..edc8eb87 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ geoapi: .PHONY: workers workers: - docker build -t $(GEOAPI_WORKERS):$(TAG) -f devops/Dockerfile.potree . + docker build -t $(GEOAPI_WORKERS):$(TAG) -f devops/Dockerfile.worker . docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):latest docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):local From 728973400410c3e95aa9aa99c1049dd6e2b48fa3 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 10:14:00 -0500 Subject: [PATCH 25/49] Fix workflow --- .github/workflows/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 652a06e5..465004fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,9 +58,13 @@ jobs: restore-keys: | ${{ runner.os }}-poetry- - name: Install Python Packages - run: poetry install + run: | + cd devops + poetry install - name: Run server-side unit tests - run: poetry run pytest + run: | + cd .. + poetry run pytest Workers_Unit_Tests: runs-on: ubuntu-latest services: From 2457296fc06cea627655c9bc6b4f7d28d4f599c8 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 10:18:37 -0500 Subject: [PATCH 26/49] Fix workflow --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 465004fd..5474b838 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,8 +63,7 @@ jobs: poetry install - name: Run server-side unit tests run: | - cd .. - poetry run pytest + poetry run pytest ../geoapi Workers_Unit_Tests: runs-on: ubuntu-latest services: From e0aebdcf3b3499ec16047583e1ce8aca725b8cf3 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:02:29 -0500 Subject: [PATCH 27/49] Remove env file usage in Makefile Using docker-compose.yml where that is hardcoded --- devops/geoapi-services/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index bc0ac0e2..a7048fd1 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,7 +1,3 @@ -ENV_FILE ?= /opt/portal/secrets.env -# get GEOAPI_TAG and HAZMAPPER_TAG to be used by docker-compose.yml -include $(ENV_FILE) - .PHONY: start start: docker-compose -f docker-compose.yml up From a77c8113320fca083c8aa4a97249135d0ab3ceb8 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:14:37 -0500 Subject: [PATCH 28/49] Pass in GEOAPI_TAG from env --- devops/geoapi-services/Makefile | 2 ++ devops/geoapi-services/docker-compose.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index a7048fd1..8a1e667e 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,7 +1,9 @@ .PHONY: start start: + echo "starting services GEOAPI_TAG=$GEOAPI_TAG" docker-compose -f docker-compose.yml up .PHONY: stop stop: + echo "stopping services (GEOAPI_TAG=$GEOAPI_TAG)" docker-compose -f docker-compose.yml down diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index cd1f7de6..de976015 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -31,6 +31,7 @@ services: environment: - FLASK_APP=/app/geoapi/app.py - ASSETS_BASE_DIR=/assets + - GEOAPI_TAG=ENV_GEOAPI_TAG stdin_open: true tty: true container_name: geoapiworkers @@ -47,6 +48,7 @@ services: environment: - FLASK_APP=/app/geoapi/app.py - ASSETS_BASE_DIR=/assets + - GEOAPI_TAG=ENV_GEOAPI_TAG stdin_open: true tty: true container_name: geoapicelerybeat @@ -68,7 +70,7 @@ services: environment: - FLASK_APP=/app/geoapi/app.py - ASSETS_BASE_DIR=/assets - + - GEOAPI_TAG=ENV_GEOAPI_TAG stdin_open: true tty: true container_name: geoapi From dbaa93a0758d30b543f144d0e79729883fe5dfb9 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:19:56 -0500 Subject: [PATCH 29/49] Remove echo --- devops/geoapi-services/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index 8a1e667e..ab6ead75 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,9 +1,8 @@ + .PHONY: start start: - echo "starting services GEOAPI_TAG=$GEOAPI_TAG" docker-compose -f docker-compose.yml up .PHONY: stop stop: - echo "stopping services (GEOAPI_TAG=$GEOAPI_TAG)" docker-compose -f docker-compose.yml down From 8549c82eca84e6f516bdc228bcfa0639053fbbff Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:21:20 -0500 Subject: [PATCH 30/49] Fix docker compose statement --- devops/geoapi-services/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index ab6ead75..38af9bd5 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,8 +1,8 @@ .PHONY: start start: - docker-compose -f docker-compose.yml up + docker compose -f docker-compose.yml up .PHONY: stop stop: - docker-compose -f docker-compose.yml down + docker compose -f docker-compose.yml down From a7b41dee395c65cf4a1bdb4d0a85e9cd9fa682d8 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:46:37 -0500 Subject: [PATCH 31/49] update docker compose and Makefile --- devops/geoapi-services/Makefile | 6 +++++- devops/geoapi-services/docker-compose.yml | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index 38af9bd5..453fec58 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,8 +1,12 @@ .PHONY: start start: - docker compose -f docker-compose.yml up + docker compose -f docker-compose.yml up -d .PHONY: stop stop: docker compose -f docker-compose.yml down + +.PHONY: pull +pull: + docker pull api workers nginx rabbitmq \ No newline at end of file diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index de976015..e943b30b 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -10,6 +10,10 @@ services: networks: - geoapi env_file: /opt/portal/conf/secrets.env + environment: + - FLASK_APP=/app/geoapi/app.py + - ASSETS_BASE_DIR=/assets + - GEOAPI_TAG=ENV_GEOAPI_TAG nginx: image: nginx From 9445199d557837ca4753c94200fa8e49c64be857 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:52:08 -0500 Subject: [PATCH 32/49] Fix MAKEFILE --- devops/geoapi-services/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index 453fec58..b9f84b07 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,6 +1,7 @@ .PHONY: start start: + echo "starting (GEOAPI_TAG=$GEOAPI_TAG) docker compose -f docker-compose.yml up -d .PHONY: stop @@ -9,4 +10,4 @@ stop: .PHONY: pull pull: - docker pull api workers nginx rabbitmq \ No newline at end of file + docker compose pull api workers nginx rabbitmq From e30f69d25e98531f599ad6e8d9d3a31d7357a1e5 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:52:16 -0500 Subject: [PATCH 33/49] Add todo --- devops/geoapi-services/secrets.sample.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devops/geoapi-services/secrets.sample.env b/devops/geoapi-services/secrets.sample.env index ed76d35a..3ff4f3da 100644 --- a/devops/geoapi-services/secrets.sample.env +++ b/devops/geoapi-services/secrets.sample.env @@ -4,6 +4,9 @@ DB_PASSWD=dev DB_HOST=${DB_HOST} RABBITMQ_DEFAULT_USER=dev RABBITMQ_DEFAULT_PASS=dev +# TODO WP-283 remove duplication; we thought about doing something like RABBITMQ_DEFAULT_PASS=$RABBITMQ_PASSWORD in docker compose, but unclear; or change our geoapi settings.py +RABBITMQ_USERNAME=dev +RABBITMQ_PASSWORD='dev1234' RABBITMQ_DEFAULT_VHOST=dev RABBITMQ_HOSTNAME=rabbitmq APP_ENV=development # "production" for STAGING and PROD? From d041516cd599faa9467d4ca944e8a044859ef1cf Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 16:52:57 -0500 Subject: [PATCH 34/49] Fix MAKEFILE again --- devops/geoapi-services/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index b9f84b07..e3c29a15 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,7 +1,7 @@ .PHONY: start start: - echo "starting (GEOAPI_TAG=$GEOAPI_TAG) + echo "starting (GEOAPI_TAG=$GEOAPI_TAG)" docker compose -f docker-compose.yml up -d .PHONY: stop From 7424b824d7e0c324546e6657414168613267ac0a Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 17:04:13 -0500 Subject: [PATCH 35/49] Add missing RABBITMQ_VHOST --- devops/geoapi-services/Makefile | 2 +- devops/geoapi-services/secrets.sample.env | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index e3c29a15..0213f5a3 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -1,7 +1,7 @@ .PHONY: start start: - echo "starting (GEOAPI_TAG=$GEOAPI_TAG)" + echo "starting (GEOAPI_TAG=${GEOAPI_TAG})" docker compose -f docker-compose.yml up -d .PHONY: stop diff --git a/devops/geoapi-services/secrets.sample.env b/devops/geoapi-services/secrets.sample.env index 3ff4f3da..06bfa1d6 100644 --- a/devops/geoapi-services/secrets.sample.env +++ b/devops/geoapi-services/secrets.sample.env @@ -3,11 +3,12 @@ DB_NAME=dev DB_PASSWD=dev DB_HOST=${DB_HOST} RABBITMQ_DEFAULT_USER=dev -RABBITMQ_DEFAULT_PASS=dev -# TODO WP-283 remove duplication; we thought about doing something like RABBITMQ_DEFAULT_PASS=$RABBITMQ_PASSWORD in docker compose, but unclear; or change our geoapi settings.py +RABBITMQ_DEFAULT_PASS='dev1234' +# TODO WP-283 remove duplication (3 things); we thought about doing something like RABBITMQ_DEFAULT_PASS=$RABBITMQ_PASSWORD in docker compose, but unclear; or change our geoapi settings.py RABBITMQ_USERNAME=dev RABBITMQ_PASSWORD='dev1234' RABBITMQ_DEFAULT_VHOST=dev +RABBITMQ_VHOST=dev RABBITMQ_HOSTNAME=rabbitmq APP_ENV=development # "production" for STAGING and PROD? TENANT="{'DESIGNSAFE': {'service_account_token': 'ABCD1234'}}" From fb422f74c8ed203fdabc6f966c6d35b2341997a0 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 26 Sep 2023 17:04:53 -0500 Subject: [PATCH 36/49] Do not run docker up in background for testing --- devops/geoapi-services/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index 0213f5a3..fe66ce94 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -2,7 +2,8 @@ .PHONY: start start: echo "starting (GEOAPI_TAG=${GEOAPI_TAG})" - docker compose -f docker-compose.yml up -d + #docker compose -f docker-compose.yml up -d + docker compose -f docker-compose.yml up .PHONY: stop stop: From 69efa060b20aaa0da2dd608cfe1861cc1edc9a62 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 09:51:27 -0500 Subject: [PATCH 37/49] Fix gitignore and improve dockerignore --- .dockerignore | 4 +++- .gitignore | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 76a78bfe..d4f51b7b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ **.pyc .git **/__pycache__ - +# Secret env file +.env +*secrets.env diff --git a/.gitignore b/.gitignore index 24eaba21..b9b8a49a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ __pycache__/ # Secret env file .env -*secret.env +*secrets.env # C extensions *.so From 68b2f280fb222c92581eb022ca86edde4f4f0e40 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 09:51:47 -0500 Subject: [PATCH 38/49] Bump rabbitmq version --- devops/docker-compose.local.yml | 2 +- devops/geoapi-services/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devops/docker-compose.local.yml b/devops/docker-compose.local.yml index a1986876..c4ca30fc 100644 --- a/devops/docker-compose.local.yml +++ b/devops/docker-compose.local.yml @@ -11,7 +11,7 @@ networks: services: rabbitmq: - image: rabbitmq + image: rabbitmq:3.12.6 networks: - geoapi environment: diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index e943b30b..9d62cbeb 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -6,7 +6,7 @@ networks: services: rabbitmq: - image: rabbitmq:3.8.18 + image: rabbitmq:3.12.6 networks: - geoapi env_file: /opt/portal/conf/secrets.env From 3a0d55c2bab177e0a84a6d3cd250ecdb89ff4c8d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 09:52:14 -0500 Subject: [PATCH 39/49] Update todo --- devops/geoapi-services/secrets.sample.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/geoapi-services/secrets.sample.env b/devops/geoapi-services/secrets.sample.env index 06bfa1d6..726c43c1 100644 --- a/devops/geoapi-services/secrets.sample.env +++ b/devops/geoapi-services/secrets.sample.env @@ -4,7 +4,7 @@ DB_PASSWD=dev DB_HOST=${DB_HOST} RABBITMQ_DEFAULT_USER=dev RABBITMQ_DEFAULT_PASS='dev1234' -# TODO WP-283 remove duplication (3 things); we thought about doing something like RABBITMQ_DEFAULT_PASS=$RABBITMQ_PASSWORD in docker compose, but unclear; or change our geoapi settings.py +# TODO https://jira.tacc.utexas.edu/browse/WP-283 remove duplication RABBITMQ_USERNAME=dev RABBITMQ_PASSWORD='dev1234' RABBITMQ_DEFAULT_VHOST=dev From e86673a8fcc0d4101e21efaf498bbde83fe722c1 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 09:52:48 -0500 Subject: [PATCH 40/49] Fix tabbitmq password variable on example secrets file --- devops/geoapi-services/secrets.sample.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/geoapi-services/secrets.sample.env b/devops/geoapi-services/secrets.sample.env index 726c43c1..004f0c44 100644 --- a/devops/geoapi-services/secrets.sample.env +++ b/devops/geoapi-services/secrets.sample.env @@ -6,7 +6,7 @@ RABBITMQ_DEFAULT_USER=dev RABBITMQ_DEFAULT_PASS='dev1234' # TODO https://jira.tacc.utexas.edu/browse/WP-283 remove duplication RABBITMQ_USERNAME=dev -RABBITMQ_PASSWORD='dev1234' +RABBITMQ_PASSWD='dev1234' RABBITMQ_DEFAULT_VHOST=dev RABBITMQ_VHOST=dev RABBITMQ_HOSTNAME=rabbitmq From 2f094b8c1bdde27f472f97e1076b62c91b316b3d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 09:55:59 -0500 Subject: [PATCH 41/49] Refactor how settings are set based on APP_ENV --- geoapi/settings.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/geoapi/settings.py b/geoapi/settings.py index 83175f1c..da48411a 100644 --- a/geoapi/settings.py +++ b/geoapi/settings.py @@ -4,22 +4,14 @@ class Config(object): DEBUG = False TESTING = False - DB_USERNAME = 'dev' - DB_NAME = 'dev' - DB_PASSWD = 'dev' - DB_HOST = 'postgres' ASSETS_BASE_DIR = os.environ.get('ASSETS_BASE_DIR', '/assets') JWT_PUB_KEY = 'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCUp/oV1vWc8/TkQSiAvTousMzOM4asB2iltr2QKozni5aVFu818MpOLZIr8LMnTzWllJvvaA5RAAdpbECb+48FjbBe0hseUdN5HpwvnH/DW8ZccGvk53I6Orq7hLCv1ZHtuOCokghz/ATrhyPq+QktMfXnRS4HrKGJTzxaCcU7OQIDAQAB' # noqa: E501 - RABBITMQ_USERNAME = 'dev' - RABBITMQ_PASSWD = 'dev' - RABBITMQ_VHOST = 'dev' - RABBITMQ_HOSTNAME = 'rabbitmq' RESTPLUS_MASK_SWAGGER = False TENANT = os.environ.get("TENANT") STREETVIEW_DIR = os.environ.get('STREETVIEW_DIR', '/assets/streetview') -class ProductionConfig(Config): +class DeployedConfig(Config): DEBUG = False DB_USERNAME = os.environ.get("DB_USERNAME") DB_NAME = os.environ.get("DB_NAME") @@ -32,13 +24,21 @@ class ProductionConfig(Config): MAPILLARY_CLIENT_ID = os.environ.get('MAPILLARY_CLIENT_ID', 'MLY|5156692464392931|4f1118aa1b06f051a44217cb56bedf79') -class DevelopmentConfig(Config): +class LocalDevelopmentConfig(Config): DEBUG = True PROPAGATE_EXCEPTIONS = False + RABBITMQ_USERNAME = 'dev' + RABBITMQ_PASSWD = 'dev' + RABBITMQ_VHOST = 'dev' + RABBITMQ_HOSTNAME = 'rabbitmq' + DB_USERNAME = 'dev' + DB_NAME = 'dev' + DB_PASSWD = 'dev' + DB_HOST = 'postgres' MAPILLARY_CLIENT_ID = os.environ.get('MAPILLARY_CLIENT_ID', 'MLY|4866220476802272|cedfb10deac752ca3ddf83997cef60a4') -class TestingConfig(Config): +class UnitTestingConfig(Config): DB_USERNAME = 'dev' DB_NAME = 'test' DB_PASSWD = 'dev' @@ -51,9 +51,9 @@ class TestingConfig(Config): APP_ENV = os.environ.get('APP_ENV', '').lower() -if APP_ENV == 'production': - settings = ProductionConfig +if APP_ENV == 'production' or APP_ENV == 'staging' or APP_ENV == 'dev': + settings = DeployedConfig elif APP_ENV == 'testing': - settings = TestingConfig + settings = UnitTestingConfig else: - settings = DevelopmentConfig + settings = LocalDevelopmentConfig From f05f5c3e022dd45290f257e27938b7d0327373e6 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 09:56:14 -0500 Subject: [PATCH 42/49] Add hazmapper to docker-compose --- devops/geoapi-services/docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index 9d62cbeb..49f730f6 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -82,3 +82,9 @@ services: command: "gunicorn -w 4 -b 0.0.0.0:8000 geoapi.app:app -k gevent --reload --timeout 300" extra_hosts: - "host.docker.internal:host-gateway" + + hazmapper: + image: taccaci/hazmapper:${HAZMAPPER_TAG} + ports: + - 80:80 + container_name: hazmapper From 4ebf1854040e0d3ecb745853cc8a8d8f7ded712c Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 11:57:16 -0500 Subject: [PATCH 43/49] Fix makefile --- devops/geoapi-services/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devops/geoapi-services/Makefile b/devops/geoapi-services/Makefile index fe66ce94..0213f5a3 100644 --- a/devops/geoapi-services/Makefile +++ b/devops/geoapi-services/Makefile @@ -2,8 +2,7 @@ .PHONY: start start: echo "starting (GEOAPI_TAG=${GEOAPI_TAG})" - #docker compose -f docker-compose.yml up -d - docker compose -f docker-compose.yml up + docker compose -f docker-compose.yml up -d .PHONY: stop stop: From e536f4be5c733f69cd5189ba56d1245a2e78c8f9 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 13:35:13 -0500 Subject: [PATCH 44/49] Expose rabbitmq port from geoapi-services --- devops/geoapi-services/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index 49f730f6..bc3ae0de 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -9,6 +9,8 @@ services: image: rabbitmq:3.12.6 networks: - geoapi + ports: + - "5672:5672" env_file: /opt/portal/conf/secrets.env environment: - FLASK_APP=/app/geoapi/app.py From 8f1314f85c5a01c7400ad94d4c27cb5cd1c302c9 Mon Sep 17 00:00:00 2001 From: Sophia Massie Date: Thu, 28 Sep 2023 13:51:37 -0500 Subject: [PATCH 45/49] Setup docker compose for workers --- devops/geoapi-services/docker-compose.yml | 17 ----------------- devops/geoapi-workers/Makefile | 12 ++++++++++++ .../geoapi-workers/docker-compose-workers.yml | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 devops/geoapi-workers/Makefile create mode 100644 devops/geoapi-workers/docker-compose-workers.yml diff --git a/devops/geoapi-services/docker-compose.yml b/devops/geoapi-services/docker-compose.yml index bc3ae0de..971ee335 100644 --- a/devops/geoapi-services/docker-compose.yml +++ b/devops/geoapi-services/docker-compose.yml @@ -27,23 +27,6 @@ services: networks: - geoapi - workers: - image: taccaci/geoapi-workers:${GEOAPI_TAG} - networks: - - geoapi - volumes: - - /assets:/assets - env_file: /opt/portal/conf/secrets.env - environment: - - FLASK_APP=/app/geoapi/app.py - - ASSETS_BASE_DIR=/assets - - GEOAPI_TAG=ENV_GEOAPI_TAG - stdin_open: true - tty: true - container_name: geoapiworkers - hostname: geoapiworkers - command: "celery -A geoapi.celery_app worker -l info" - celerybeat: image: taccaci/geoapi-workers:${GEOAPI_TAG} networks: diff --git a/devops/geoapi-workers/Makefile b/devops/geoapi-workers/Makefile new file mode 100644 index 00000000..d7e720b2 --- /dev/null +++ b/devops/geoapi-workers/Makefile @@ -0,0 +1,12 @@ +.PHONY: start +start: + echo "starting (GEOAPI_TAG=${GEOAPI_TAG})" + docker compose -f docker-compose.yml up -d + +.PHONY: stop +stop: + docker compose -f docker-compose.yml down + +.PHONY: pull +pull: + docker compose pull workers \ No newline at end of file diff --git a/devops/geoapi-workers/docker-compose-workers.yml b/devops/geoapi-workers/docker-compose-workers.yml new file mode 100644 index 00000000..7a491d39 --- /dev/null +++ b/devops/geoapi-workers/docker-compose-workers.yml @@ -0,0 +1,19 @@ +--- +version: "3" +services: + workers: + image: taccaci/geoapi-workers:${GEOAPI_TAG} + networks: + - geoapi + volumes: + - /assets:/assets + env_file: /opt/portal/conf/secrets.env + environment: + - FLASK_APP=/app/geoapi/app.py + - ASSETS_BASE_DIR=/assets + - GEOAPI_TAG=ENV_GEOAPI_TAG + stdin_open: true + tty: true + container_name: geoapiworkers + hostname: geoapiworkers + command: "celery -A geoapi.celery_app worker -l info" \ No newline at end of file From 11cb1e59078a3494c238a608e9a8095cd2f65c7b Mon Sep 17 00:00:00 2001 From: Sophia Massie Date: Thu, 28 Sep 2023 14:02:55 -0500 Subject: [PATCH 46/49] Edit name of worker docker compose --- .../geoapi-workers/docker-compose-workers.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 devops/geoapi-workers/docker-compose-workers.yml diff --git a/devops/geoapi-workers/docker-compose-workers.yml b/devops/geoapi-workers/docker-compose-workers.yml deleted file mode 100644 index 7a491d39..00000000 --- a/devops/geoapi-workers/docker-compose-workers.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -version: "3" -services: - workers: - image: taccaci/geoapi-workers:${GEOAPI_TAG} - networks: - - geoapi - volumes: - - /assets:/assets - env_file: /opt/portal/conf/secrets.env - environment: - - FLASK_APP=/app/geoapi/app.py - - ASSETS_BASE_DIR=/assets - - GEOAPI_TAG=ENV_GEOAPI_TAG - stdin_open: true - tty: true - container_name: geoapiworkers - hostname: geoapiworkers - command: "celery -A geoapi.celery_app worker -l info" \ No newline at end of file From 0840e27eb729b1f5e62bbe3912706e96abff2e59 Mon Sep 17 00:00:00 2001 From: Sophia Massie Date: Thu, 28 Sep 2023 14:05:44 -0500 Subject: [PATCH 47/49] Changed name for docker compose --- devops/geoapi-workers/docker-compose.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 devops/geoapi-workers/docker-compose.yml diff --git a/devops/geoapi-workers/docker-compose.yml b/devops/geoapi-workers/docker-compose.yml new file mode 100644 index 00000000..7a491d39 --- /dev/null +++ b/devops/geoapi-workers/docker-compose.yml @@ -0,0 +1,19 @@ +--- +version: "3" +services: + workers: + image: taccaci/geoapi-workers:${GEOAPI_TAG} + networks: + - geoapi + volumes: + - /assets:/assets + env_file: /opt/portal/conf/secrets.env + environment: + - FLASK_APP=/app/geoapi/app.py + - ASSETS_BASE_DIR=/assets + - GEOAPI_TAG=ENV_GEOAPI_TAG + stdin_open: true + tty: true + container_name: geoapiworkers + hostname: geoapiworkers + command: "celery -A geoapi.celery_app worker -l info" \ No newline at end of file From 1c323df2c55793d05a7faffdd004e5df739236a6 Mon Sep 17 00:00:00 2001 From: Sophia Massie Date: Thu, 28 Sep 2023 14:10:23 -0500 Subject: [PATCH 48/49] Removed unused network for workers --- devops/geoapi-workers/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/devops/geoapi-workers/docker-compose.yml b/devops/geoapi-workers/docker-compose.yml index 7a491d39..bb7e614b 100644 --- a/devops/geoapi-workers/docker-compose.yml +++ b/devops/geoapi-workers/docker-compose.yml @@ -3,8 +3,6 @@ version: "3" services: workers: image: taccaci/geoapi-workers:${GEOAPI_TAG} - networks: - - geoapi volumes: - /assets:/assets env_file: /opt/portal/conf/secrets.env From cd1c3e38e90b1c001ab14b8a6cd1ce6da89f27b8 Mon Sep 17 00:00:00 2001 From: Sophia Massie Date: Thu, 28 Sep 2023 16:19:54 -0500 Subject: [PATCH 49/49] Change api port to 8000 --- devops/geoapi-services/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/geoapi-services/nginx.conf b/devops/geoapi-services/nginx.conf index 693e5e10..cb385510 100644 --- a/devops/geoapi-services/nginx.conf +++ b/devops/geoapi-services/nginx.conf @@ -34,7 +34,7 @@ http { return 204; } rewrite ^/api(.*) /$1 break; - proxy_pass http://geoapi:5000; + proxy_pass http://geoapi:8000; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade';