Merge pull request #99 from EyeSeeTea/development

Release 1.10.0

.flaskenv

@@ -0,0 +1,2 @@
+FLASK_ENV=production
+FLASK_APP=src/d2_docker/api/main.py

.flaskenv.secret.template

@@ -0,0 +1,2 @@
+HARBOR_USER=user
+HARBOR_PASSWORD=1234

.gitignore

@@ -107,3 +107,5 @@ venv.bak/
 *.code-workspace
 .vscode
 *.war
+
+.flaskenv.secret

Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y vim netcat git docker.io docker-compose python3 python3-pip
+COPY . /app/d2-docker
+RUN pip3 install /app/d2-docker
+
+ENTRYPOINT ["/app/d2-docker/docker-container/start.sh"]
+CMD []

README.md

@@ -1,10 +1,10 @@
 ## Requirements
 
--   Operating System: GNU/Linux or Windows 10.
--   Python >= 3.5 (with setuptools)
--   Docker >= 18
--   Docker compose >= 1.17
--   RAM memory: At least 4Gb for instance, preferrably 8Gb.
+- Operating System: GNU/Linux or Windows 10.
+- Python >= 3.5 (with setuptools)
+- Docker >= 18
+- Docker compose >= 1.17
+- RAM memory: At least 4Gb for instance, preferrably 8Gb.
 
 On Ubuntu 18.04:
 
@@ -14,11 +14,10 @@ $ sudo apt install docker.io docker-compose python3 python3-setuptools
 
 On Windows 10:
 
--   Install Python: https://www.python.org/downloads
--   Install Docker Desktop: https://docs.docker.com/docker-for-windows/install
--   Activate WSL2 (this may require to install some other dependencies):
-![image (20)](https://user-images.githubusercontent.com/6850223/138077958-3fb8a9a8-e829-495a-9b25-f0e347e411d1.png)
-
+- Install Python: https://www.python.org/downloads
+- Install Docker Desktop: https://docs.docker.com/docker-for-windows/install
+- Activate WSL2 (this may require to install some other dependencies):
+  ![image (20)](https://user-images.githubusercontent.com/6850223/138077958-3fb8a9a8-e829-495a-9b25-f0e347e411d1.png)
 
 ## Install
 
@@ -74,18 +73,18 @@ $ d2-docker start eyeseetea/dhis2-data:2.30-sierra
 
 Some notes:
 
--   A d2-docker instance is composed of 4 containers: `dhis2-data` (database + apps), `dhis2-core` (tomcat + dhis.war), `postgis` (postgres with postgis support) and `nginx` (web server).
--   By default, the image `dhis2-core` from the same organisation will be used, keeping the first part of the tag (using `-` as separator). For example: `eyeseetea/dhis2-data:2.30-sierra` will use core `eyeseetea/dhis2-core:2.30`. If you need a custom image to be used, use `--core-image= eyeseetea/dhis2-core:2.30-custom`.
--   Once started, you can connect to the DHIS2 instance (`http://localhost:PORT`) where _PORT_ is the first available port starting from 8080. You can run many images at the same time, but not the same image more than once. You can specify the port with option `-p PORT`.
--   Use option `--pull` to overwrite the local images with the images in the hub.
--   Use option `--detach` to run the container in the background.
--   Use option `--deploy-path` to run the container with a deploy path namespace (i.e: `--deploy-path=dhis2` serves `http://localhost:8080/dhis2`)
--   Use option `-k`/`--keep-containers` to re-use existing docker containers, so data from the previous run will be kept.
--   Use option `-auth` to pass the instance authentication (`USER:PASS`). It will be used to call post-tomcat scripts.
--   Use option `--run-sql=DIRECTORY` to run SQL files (.sql, .sql.gz or .dump files) after the DB has been initialized.
--   Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to.
--   Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`)
--   Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used.
+- A d2-docker instance is composed of 4 containers: `dhis2-data` (database + apps), `dhis2-core` (tomcat + dhis.war), `postgis` (postgres with postgis support) and `nginx` (web server).
+- By default, the image `dhis2-core` from the same organisation will be used, keeping the first part of the tag (using `-` as separator). For example: `eyeseetea/dhis2-data:2.30-sierra` will use core `eyeseetea/dhis2-core:2.30`. If you need a custom image to be used, use `--core-image= eyeseetea/dhis2-core:2.30-custom`.
+- Once started, you can connect to the DHIS2 instance (`http://localhost:PORT`) where _PORT_ is the first available port starting from 8080. You can run many images at the same time, but not the same image more than once. You can specify the port with option `-p PORT`.
+- Use option `--pull` to overwrite the local images with the images in the hub.
+- Use option `--detach` to run the container in the background.
+- Use option `--deploy-path` to run the container with a deploy path namespace (i.e: `--deploy-path=dhis2` serves `http://localhost:8080/dhis2`)
+- Use option `-k`/`--keep-containers` to re-use existing docker containers, so data from the previous run will be kept.
+- Use option `-auth` to pass the instance authentication (`USER:PASS`). It will be used to call post-tomcat scripts.
+- Use option `--run-sql=DIRECTORY` to run SQL files (.sql, .sql.gz or .dump files) after the DB has been initialized.
+- Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to.
+- Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`)
+- Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used.
 
 #### Custom DHIS2 dhis.conf
 
@@ -258,11 +257,11 @@ $ d2-docker upgrade \
 
 Migration folder `upgrade-sierra` should then contain data to be used in each intermediate upgrade version. Supported migration data:
 
--   DHIS2 war file: `dhis.war` (if not specified, it will be download from the releases page)
--   DHIS2 home files: `dhis2-home/`
--   Shell scripts (pre-tomcat): `*.sh`
--   Shell scripts (post-tomcat): `post-*.sh`
--   SQL files: `*.sql`
+- DHIS2 war file: `dhis.war` (if not specified, it will be download from the releases page)
+- DHIS2 home files: `dhis2-home/`
+- Shell scripts (pre-tomcat): `*.sh`
+- Shell scripts (post-tomcat): `post-*.sh`
+- SQL files: `*.sql`
 
 A full example might look:
 
@@ -303,3 +302,47 @@ $ docker image prune
 ```
 $ docker system prune -a --volumes
 ```
+
+## Development
+
+### Run d2-docker from sources
+
+```
+$ ./d2-docker-dev.sh
+```
+
+### Dockerized d2-docker (d2-container)
+
+Create a dockerized d2-docker:
+
+```
+$ bash build-docker-container.sh
+```
+
+### API Server
+
+Start Flask server in development mode:
+
+```
+$ FLASK_ENV=development flask run
+```
+
+Usage examples:
+
+```
+$ curl http://localhost:5000/version
+
+$ curl  -H "Content-Type: application/json" -sS http://localhost:5000/instances/start -X POST \
+    -d '{"image": "docker.eyeseetea.com/samaritans/dhis2-data:2.36.8-sp-ip-training", "port": 8080, "detach": true}'
+```
+
+Currently, there are no API docs nor params validations. For each command `src/d2_docker/commands/COMMAND.py`, check function `setup` to see the supported parameters.
+
+The API server provides a proxy to Harbor to bypass CORS issues. Configure first the harbor authentication file (`.flaskenv.secret`):
+
+```
+$ cp .flaskenv.secret.template .flaskenv.secret
+$ # edit .flaskenv.secret and restart flask server
+
+$ curl -sS 'http://localhost:5000/harbor/https://docker.eyeseetea.com/api/v2.0/quotas/1' | jq
+```

build-docker-container.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e -u -o pipefail
+
+image_name="eyeseetea/d2-docker-container"
+version=$(./d2-docker-dev.sh version | grep "^d2-docker" | awk '{print $3}')
+docker build -t "$image_name:$version" .
+docker image tag "$image_name:$version" "$image_name:latest"

d2-docker-dev.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export PYTHONPATH=src
+exec python3 $PYTHONPATH/d2_docker/cli.py "$@"

d2-docker-in-container.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec docker exec docker-container-d2-docker-1 d2-docker "$@"

docker-container/docker-compose.yml

@@ -0,0 +1,13 @@
+version: "3"
+services:
+    d2-docker:
+        image: eyeseetea/d2-docker-container:latest
+        restart: unless-stopped
+        ports:
+            - 5000:5000
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+            - ./shared:/app/shared
+        environment:
+            ROOT_PATH: $PWD/shared/d2-docker
+            VOLUMES_OWNERSHIP: ${HOST_UID}:${HOST_GID}

docker-container/start.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -x
+set -e -u -o pipefail
+
+get_script_dir() {
+    (cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+}
+
+expose_d2_docker_to_host() {
+    script_dir=$(get_script_dir)
+    cp -av "$script_dir/../src/d2_docker/." "/app/shared/d2-docker"
+}
+
+start_api() {
+    d2-docker api start
+}
+
+fix_volumes_ownership() {
+    if test "$VOLUMES_OWNERSHIP" -a "$VOLUMES_OWNERSHIP" != ":"; then
+        chown -R "$VOLUMES_OWNERSHIP" /app/shared/
+    fi
+}
+
+main() {
+    expose_d2_docker_to_host
+    fix_volumes_ownership
+    start_api
+}
+
+main

requirements.txt

@@ -0,0 +1,3 @@
+Flask==2.0.3
+python-dotenv
+Flask_Cors==3.0.10

setup.py

@@ -4,7 +4,7 @@
 
 setuptools.setup(
     name="d2_docker",
-    version="1.9.0",
+    version="1.10.0",
     description="Dockers for DHIS2 instances",
     long_description=open("README.md", encoding="utf-8").read(),
     keywords=["python"],
@@ -25,5 +25,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
     ],
-    entry_points={"console_scripts": ["d2-docker=d2_docker.cli:main"]},
+    entry_points={"console_scripts": [
+        "d2-docker=d2_docker.cli:main",
+    ]},
 )

src/d2_docker/api/api_utils.py

@@ -0,0 +1,50 @@
+import os
+import base64
+from d2_docker.commands import list_
+from flask import jsonify
+from dotenv import dotenv_values
+
+
+class Struct(object):
+    def __init__(self, dictionary):
+        self.__dict__.update(dictionary)
+
+    def __getattr__(self, item):
+        return None
+
+    def __repr__(self):
+        pairs = ", ".join("{}={}".format(k, repr(v)) for (k, v) in self.__dict__.items())
+        return "Struct(" + pairs + ")"
+
+
+def get_args_from_request(request):
+    body = request.get_json()
+    args = Struct(body)
+    return args
+
+
+def get_auth_headers(user, password):
+    encoded_auth = base64.b64encode((user + ":" + password).encode()).decode()
+    return {"Authorization": "Basic " + encoded_auth}
+
+
+def get_container(name):
+    containers = list_.get_containers()
+    return next((c for c in containers if c["name"] == name), None)
+
+
+def success():
+    return jsonify(dict(status="SUCCESS"))
+
+
+def server_error(message, status=500):
+    body = jsonify(dict(status="ERROR", error=message))
+    return (body, status)
+
+
+def get_config():
+    return {
+        **dotenv_values(".flaskenv"),
+        **dotenv_values(".flaskenv.secret"),
+        **os.environ,
+    }