From a92962b2df7286f4a11e7c4765eb3054a54e10e4 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Thu, 7 Jul 2022 11:05:46 +0200 Subject: [PATCH 1/5] Fix content-type response for proxied harbor requests --- src/d2_docker/api/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2_docker/api/main.py b/src/d2_docker/api/main.py index 2b555f0..9d1737c 100644 --- a/src/d2_docker/api/main.py +++ b/src/d2_docker/api/main.py @@ -103,7 +103,7 @@ def proxy_request_to_url(request, url, new_headers=None): forward_request = http_request(url, headers=headers) response = stream_with_context(forward_request.iter_content()) - return Response(response, content_type=request.content_type) + return Response(response, content_type=forward_request.headers.get("Content-Type")) @api.route("/harbor/", methods=["GET", "POST", "PUT", "DELETE"]) From 40b16a46b41eb018fd39599645790cca783f109f Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Thu, 7 Jul 2022 14:39:04 +0200 Subject: [PATCH 2/5] Safe request.json --- src/d2_docker/api/api_utils.py | 4 ++-- src/d2_docker/api/main.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/d2_docker/api/api_utils.py b/src/d2_docker/api/api_utils.py index 2c57efd..fe2e735 100644 --- a/src/d2_docker/api/api_utils.py +++ b/src/d2_docker/api/api_utils.py @@ -44,7 +44,7 @@ def server_error(message, status=500): def get_config(): return { - **dotenv_values(".flaskenv"), - **dotenv_values(".flaskenv.secret"), + **dotenv_values(".flaskenv", verbose=True), + **dotenv_values(".flaskenv.secret", verbose=True), **os.environ, } diff --git a/src/d2_docker/api/main.py b/src/d2_docker/api/main.py index 9d1737c..b34ba43 100644 --- a/src/d2_docker/api/main.py +++ b/src/d2_docker/api/main.py @@ -5,7 +5,7 @@ from flask import Response, stream_with_context from flask_cors import CORS -from werkzeug.exceptions import HTTPException +from werkzeug.exceptions import HTTPException, BadRequest from d2_docker import utils from d2_docker.commands import version, list_, start, stop, logs, commit, push, pull @@ -89,14 +89,22 @@ def rm_instance(): return success() +def get_request_json(request): + try: + return request.json + except BadRequest: + return None + + def proxy_request_to_url(request, url, new_headers=None): base_headers = utils.dict_remove(dict(request.headers), "Host") headers = utils.dict_merge(base_headers, new_headers or {}) method = request.method.lower() http_request = getattr(requests, method) + request_json = get_request_json(request) - if request.json: - forward_request = http_request(url, json=request.json, headers=headers) + if request_json: + forward_request = http_request(url, json=request_json, headers=headers) elif request.form: forward_request = http_request(url, data=request.form.to_dict(), headers=headers) else: From bf7d52e01bb3afce678a9f11282fa93ac17e0171 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Fri, 8 Jul 2022 09:53:30 +0200 Subject: [PATCH 3/5] Add user config folder to dotenv loading paths --- src/d2_docker/api/api_utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/d2_docker/api/api_utils.py b/src/d2_docker/api/api_utils.py index fe2e735..feddf85 100644 --- a/src/d2_docker/api/api_utils.py +++ b/src/d2_docker/api/api_utils.py @@ -1,6 +1,7 @@ import os import base64 from d2_docker.commands import list_ +from d2_docker import utils from flask import jsonify from dotenv import dotenv_values @@ -42,9 +43,22 @@ def server_error(message, status=500): return (body, status) +def get_from_dotenv(name, directories): + output = {} + for directory in directories: + path1 = os.path.join(directory, name) + path2 = os.path.expanduser(path1) + utils.logger.info("Try config path: {}".format(path2)) + value = dotenv_values(path2, verbose=True) + output.update(value) + return output + + def get_config(): + directories = ["~/.config/d2-docker"] + return { - **dotenv_values(".flaskenv", verbose=True), - **dotenv_values(".flaskenv.secret", verbose=True), + **get_from_dotenv(".flaskenv", directories), + **get_from_dotenv(".flaskenv.secret", directories), **os.environ, } From 142e5390bd9ad8df17b3524c9b451a9504a14c7f Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Fri, 8 Jul 2022 11:28:54 +0200 Subject: [PATCH 4/5] Cache config --- README.md | 8 +++++--- ...v.secret.template => flaskenv.secret.template | 0 src/d2_docker/api/api_utils.py | 16 ++++++++++------ src/d2_docker/api/main.py | 11 +++-------- src/d2_docker/commands/api.py | 3 +-- 5 files changed, 19 insertions(+), 19 deletions(-) rename .flaskenv.secret.template => flaskenv.secret.template (100%) diff --git a/README.md b/README.md index 57f5666..a3c1fea 100644 --- a/README.md +++ b/README.md @@ -338,11 +338,13 @@ $ curl -H "Content-Type: application/json" -sS http://localhost:5000/instances/ Currently, there are no API docs nor params validations. For each command `src/d2_docker/commands/COMMAND.py`, check function `setup` to see the supported parameters. -The API server provides a proxy to Harbor to bypass CORS issues. Configure first the harbor authentication file (`.flaskenv.secret`): +The API server provides a proxy to Harbor to bypass CORS issues. Configure first the harbor authentication file: ``` -$ cp .flaskenv.secret.template .flaskenv.secret -$ # edit .flaskenv.secret and restart flask server +$ cp flaskenv.secret.template flaskenv.secret +$ # Edit flaskenv.secret +$ mkdir -p ~/.config/d2-docker/ +$ cp flaskenv.secret ~/.config/d2-docker/ $ curl -sS 'http://localhost:5000/harbor/https://docker.eyeseetea.com/api/v2.0/quotas/1' | jq ``` diff --git a/.flaskenv.secret.template b/flaskenv.secret.template similarity index 100% rename from .flaskenv.secret.template rename to flaskenv.secret.template diff --git a/src/d2_docker/api/api_utils.py b/src/d2_docker/api/api_utils.py index feddf85..7f49436 100644 --- a/src/d2_docker/api/api_utils.py +++ b/src/d2_docker/api/api_utils.py @@ -1,7 +1,6 @@ import os import base64 from d2_docker.commands import list_ -from d2_docker import utils from flask import jsonify from dotenv import dotenv_values @@ -48,17 +47,22 @@ def get_from_dotenv(name, directories): for directory in directories: path1 = os.path.join(directory, name) path2 = os.path.expanduser(path1) - utils.logger.info("Try config path: {}".format(path2)) value = dotenv_values(path2, verbose=True) output.update(value) return output +config = None + + def get_config(): - directories = ["~/.config/d2-docker"] + global config + if config: + return config - return { - **get_from_dotenv(".flaskenv", directories), - **get_from_dotenv(".flaskenv.secret", directories), + directories = ["~/.config/d2-docker"] + config = { + **get_from_dotenv("flaskenv.secret", directories), **os.environ, } + return config diff --git a/src/d2_docker/api/main.py b/src/d2_docker/api/main.py index b34ba43..6792339 100644 --- a/src/d2_docker/api/main.py +++ b/src/d2_docker/api/main.py @@ -116,6 +116,7 @@ def proxy_request_to_url(request, url, new_headers=None): @api.route("/harbor/", methods=["GET", "POST", "PUT", "DELETE"]) def proxy(url): + config = get_config() user = config.get("HARBOR_USER") password = config.get("HARBOR_PASSWORD") if not user or not password: @@ -137,12 +138,6 @@ def internal_error(error): return server_error(contents, status=status) -config = get_config() - - def run(args): - api.run(host=args.host or "0.0.0.0", port=args.port or 5000) - - -if __name__ == "__main__": - run() + get_config() + api.run(host=args.host or "127.0.0.1", port=args.port or 5000) diff --git a/src/d2_docker/commands/api.py b/src/d2_docker/commands/api.py index 33eef81..2187603 100644 --- a/src/d2_docker/commands/api.py +++ b/src/d2_docker/commands/api.py @@ -6,12 +6,11 @@ def setup(parser): subparsers = parser.add_subparsers(dest="api_command") start_parser = subparsers.add_parser("start") - start_parser.add_argument("--host", type=int, help="Listen host") + start_parser.add_argument("--host", type=str, help="Listen host") start_parser.add_argument("-p", "--port", type=int, help="Listen port") def run(args): - print(args) if args.api_command == "start": main.run(args) elif args.api_command == "stop": From 133b34df2871dc1343c37ff07ce6e7041b09d07f Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 11 Jul 2022 14:56:12 +0200 Subject: [PATCH 5/5] Use init-done flag in core container (fixes automatic restart) --- src/d2_docker/config/dhis2-core-start.sh | 25 +++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/d2_docker/config/dhis2-core-start.sh b/src/d2_docker/config/dhis2-core-start.sh index 4db0573..92231c8 100755 --- a/src/d2_docker/config/dhis2-core-start.sh +++ b/src/d2_docker/config/dhis2-core-start.sh @@ -122,14 +122,33 @@ wait_for_tomcat() { done } +INIT_DONE_FILE="/tmp/dhis2-core-start.done" + +is_init_done() { + test -e "$INIT_DONE_FILE" +} + +init_done() { + touch "$INIT_DONE_FILE" +} + run() { local host=$1 psql_port=$2 + setup_tomcat copy_apps copy_documents - wait_for_postgres - run_sql_files || true - run_pre_scripts || true + + if is_init_done; then + debug "Container: already configured. Skip DB load" + else + debug "Container: clean. Load DB" + wait_for_postgres + run_sql_files || true + run_pre_scripts || true + init_done + fi + start_tomcat & wait_for_tomcat run_post_scripts || true