Merge pull request #57 from EyeSeeTea/development

Development

MANIFEST.in

@@ -4,3 +4,4 @@ include requirements.txt
 include src/d2_docker/docker-compose.yml
 recursive-include src/d2_docker/config *
 recursive-include src/d2_docker/images *
+recursive-include src/d2_docker/.empty *

README.md

@@ -76,10 +76,42 @@ Some notes:
 -   By default, the image `dhis2-core` from the same organisation will be used, keeping the first part of the tag (using `-` as separator). For example: `eyeseetea/dhis2-data:2.30-sierra` will use core `eyeseetea/dhis2-core:2.30`. If you need a custom image to be used, use `--core-image= eyeseetea/dhis2-core:2.30-custom`.
 -   Once started, you can connect to the DHIS2 instance (`http://localhost:PORT`) where _PORT_ is the first available port starting from 8080. You can run many images at the same time, but not the same image more than once. You can specify the port with option `-p PORT`.
 -   Use option `--pull` to overwrite the local images with the images in the hub.
--   Use option `--detach` to run containers in the background.
--   Use option `-k`/`--keep-containers` to re-use existing containers, so data from the previous run will be kept.
+-   Use option `--detach` to run the container in the background.
+-   Use option `--deploy-path` to run the container with a deploy path namespace (i.e: `--deploy-path=dhis2` serves `http://localhost:8080/dhis2`)
+-   Use option `-k`/`--keep-containers` to re-use existing docker containers, so data from the previous run will be kept.
+-   Use option `-auth` to pass the instance authentication (`USER:PASS`). It will be used to call post-tomcat scripts.
 -   Use option `--run-sql=DIRECTORY` to run SQL files (.sql, .sql.gz or .dump files) after the DB has been initialized.
--   Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080`, regardless of the public port that the instance is exposed to. Of course, this endpoint is only available for post-scripts.
+-   Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to.
+
+#### Custom Tomcat server.xml
+
+Copy the default [server.xml](https://github.com/EyeSeeTea/d2-docker/blob/master/src/d2_docker/config/server.xml) and use it as a template to create your own configuration. Then pass it to the `start` command:
+
+```
+$ d2-docker start --tomcat-server-xml=server-xml.xml ...
+```
+
+Note that you should not change the catalina connector port (8080, by default). A typical configuration to use _https_ would look like this:
+
+```
+<Server port="8005" shutdown="SHUTDOWN">
+    ...
+    <Service name="Catalina">
+        <Connector
+            port="8080"
+            protocol="HTTP/1.1"
+            proxyPort="443"
+            scheme="https"
+            secure="true"
+            proxyName="some-host.org"
+            connectionTimeout="20000"
+            URIEncoding="UTF-8"
+            relaxedQueryChars='\ { } | [ ]'
+            redirectPort="8443"
+        />
+    ...
+/>
+```
 
 ### Show logs for running containers
 
@@ -214,11 +246,11 @@ $ d2-docker upgrade \
 
 Migration folder `upgrade-sierra` should then contain data to be used in each intermediate upgrade version. Supported migration data:
 
-- DHIS2 war file: `dhis.war` (if not specified, it will be download from the releases page)
-- DHIS2 home files: `dhis2-home/`
-- Shell scripts (pre-tomcat): `*.sh`
-- Shell scripts (post-tomcat): `post-*.sh`
-- SQL files: `*.sql`
+-   DHIS2 war file: `dhis.war` (if not specified, it will be download from the releases page)
+-   DHIS2 home files: `dhis2-home/`
+-   Shell scripts (pre-tomcat): `*.sh`
+-   Shell scripts (post-tomcat): `post-*.sh`
+-   SQL files: `*.sql`
 
 A full example might look:
 

setup.py

@@ -4,7 +4,7 @@
 
 setuptools.setup(
     name="d2_docker",
-    version="1.0.6",
+    version="1.0.8",
     description="Dockers for DHIS2 instances",
     long_description=open("README.md", encoding="utf-8").read(),
     keywords=["python"],

src/d2_docker/cli.py

@@ -18,6 +18,7 @@
     run_sql,
     create,
     upgrade,
+    version
 )
 
 COMMAND_MODULES = [
@@ -35,6 +36,7 @@
     run_sql,
     create,
     upgrade,
+    version,
 ]
 
 def get_parser():
@@ -47,7 +49,7 @@ def get_parser():
     parser.add_argument(
         "--log-level",
         metavar="NOTSET | DEBUG | INFO | WARNING | ERROR | CRITICAL",
-        default="DEBUG",
+        default="INFO",
         help="Run command with the given log level",
     )
     subparsers = parser.add_subparsers(help="Subcommands", dest="command")

src/d2_docker/commands/start.py

@@ -1,22 +1,29 @@
 import os
 import re
 
+import d2_docker
 from d2_docker import utils
 
 DESCRIPTION = "Start a container from an existing dhis2-data Docker image or from an exported file"
 
 
 def setup(parser):
+    d2_docker_path = os.path.abspath(d2_docker.__path__[0])
+    server_xml_path = os.path.join(d2_docker_path, "config", "server.xml")
+    server_xml_help = "Use a custom Tomcat server.xml file. Template: {0}".format(server_xml_path)
+
     parser.add_argument(
         "image_or_file", metavar="IMAGE_OR_EXPORT_FILE", help="Docker image or exported file"
     )
     utils.add_core_image_arg(parser)
+    parser.add_argument("--auth", metavar="USER:PASSWORD", help="Dhis2 instance authentication")
     parser.add_argument(
         "-d", "--detach", action="store_true", help="Run containers on the background"
     )
     parser.add_argument(
         "-k", "--keep-containers", action="store_true", help="Keep existing containers"
     )
+    parser.add_argument("--tomcat-server-xml", metavar="FILE", help=server_xml_help)
     parser.add_argument("--run-sql", metavar="DIRECTORY", help="Run .sql[.gz] files in directory")
     parser.add_argument(
         "--run-scripts",
@@ -25,6 +32,7 @@ def setup(parser):
     )
     parser.add_argument("--pull", action="store_true", help="Force a pull from docker hub")
     parser.add_argument("-p", "--port", type=int, metavar="N", help="Set Dhis2 instance port")
+    parser.add_argument("--deploy-path", type=str, help="Set Tomcat context.path")
 
 
 def run(args):
@@ -73,6 +81,8 @@ def start(args, image_name):
         bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None]
     )
 
+    deploy_path = ("/" + re.sub("^/*", "", args.deploy_path) if args.deploy_path else "")
+
     with utils.stop_docker_on_interrupt(image_name, core_image):
         utils.run_docker_compose(
             ["up", *up_args],
@@ -82,6 +92,9 @@ def start(args, image_name):
             load_from_data=override_containers,
             post_sql_dir=args.run_sql,
             scripts_dir=args.run_scripts,
+            deploy_path=deploy_path,
+            dhis2_auth=args.auth,
+            tomcat_server=args.tomcat_server_xml
         )
 
     if args.detach:

src/d2_docker/commands/version.py

@@ -0,0 +1,20 @@
+from d2_docker import utils
+import pkg_resources  # part of setuptools
+
+DESCRIPTION = "Show version"
+
+
+def setup(parser):
+    pass
+
+
+def run(args):
+    utils.run(["docker", "-v"])
+    utils.run(["docker-compose", "-v"])
+    resource = pkg_resources.require("d2-docker")
+    if resource:
+        version = pkg_resources.require("d2-docker")[0].version
+        print("d2-docker version {}".format(version))
+    else:
+        print("Cannot get d2-docker version")
+

src/d2_docker/config/dhis2-core-start.sh

@@ -9,11 +9,14 @@ set -e -u -o pipefail
 #   - Run post-tomcat shell scripts
 #
 
-# Global LOAD_FROM_DATA="yes" | "no"
+# Global: LOAD_FROM_DATA="yes" | "no"
+# Global: DEPLOY_PATH=string
+# Global: DHIS2_AUTH=string
 
 export PGPASSWORD="dhis"
 
-dhis2_url="http://localhost:8080"
+dhis2_url="http://localhost:8080/$DEPLOY_PATH"
+dhis2_url_with_auth="http://$DHIS2_AUTH@localhost:8080/$DEPLOY_PATH"
 psql_cmd="psql -v ON_ERROR_STOP=0 --quiet -h db -U dhis dhis2"
 pgrestore_cmd="pg_restore -h db -U dhis -d dhis2"
 configdir="/config"
@@ -23,6 +26,7 @@ root_db_path="/data/db"
 post_db_path="/data/db/post"
 source_apps_path="/data/apps"
 dest_apps_path="/DHIS2_home/files/"
+tomcat_conf_dir="/usr/local/tomcat/conf"
 
 debug() {
     echo "[dhis2-core-start] $*" >&2
@@ -53,14 +57,14 @@ run_sql_files() {
 run_pre_scripts() {
     find "$scripts_dir" -type f -name '*.sh' ! \( -name 'post*' \) | sort | while read -r path; do
         debug "Run pre-tomcat script: $path"
-        (cd "$(dirname "$path")" && bash "$path")
+        (cd "$(dirname "$path")" && bash -x "$path")
     done
 }
 
 run_post_scripts() {
     find "$scripts_dir" -type f -name '*.sh' -name 'post*' | sort | while read -r path; do
         debug "Run post-tomcat script: $path"
-        (cd "$(dirname "$path")" && bash "$path")
+        (cd "$(dirname "$path")" && bash -x "$path" "$dhis2_url_with_auth")
     done
 }
 
@@ -73,9 +77,11 @@ copy_apps() {
 }
 
 setup_tomcat() {
+    debug "Setup tomcat"
     cp -v "$configdir/DHIS2_home/dhis.conf" /DHIS2_home/dhis.conf
     cp -v $homedir/* /DHIS2_home/ || true
-    cp -v "$configdir/server.xml" /usr/local/tomcat/conf/server.xml
+    cp -v "$configdir/server.xml" "$tomcat_conf_dir/server.xml"
+    find "$configdir/override/tomcat/" -maxdepth 1 -type f -size +0 -exec cp -v {} "$tomcat_conf_dir/" \;
 }
 
 wait_for_postgres() {
@@ -92,7 +98,7 @@ start_tomcat() {
 
 wait_for_tomcat() {
     debug "Waiting for Tomcat to start: $dhis2_url"
-    while ! curl -sS -i "$dhis2_url" 2>/dev/null | grep "Location: .*redirect.action"; do
+    while ! curl -sS -i "$dhis2_url" 2>/dev/null | grep "^Location"; do
         sleep 1
     done
 }

src/d2_docker/config/server.xml

@@ -18,14 +18,20 @@
     </GlobalNamingResources>
 
     <Service name="Catalina">
-        <Connector port="8080" protocol="HTTP/1.1"
-            connectionTimeout="20000"
+        <Connector
+            protocol="HTTP/1.1"
+            port="8080"
             redirectPort="8443"
+            connectionTimeout="20000"
             URIEncoding="UTF-8"
             relaxedQueryChars='\ { } | [ ]'
         />
 
-        <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
+        <Connector
+            protocol="AJP/1.3"
+            port="8009"
+            redirectPort="8443"
+        />
 
         <Engine name="Catalina" defaultHost="localhost">
             <Realm className="org.apache.catalina.realm.LockOutRealm">

src/d2_docker/docker-compose.yml

@@ -8,12 +8,15 @@ services:
             - home:/DHIS2_home
             - ./config:/config
             - data:/data
+            - "${TOMCAT_SERVER}:/config/override/tomcat/server.xml"
             - "${SCRIPTS_DIR}:/data/scripts"
             - "${POST_SQL_DIR}:/data/db/post"
         environment:
-            CATALINA_OPTS: "-Dcontext.path="
+            CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH}"
             JAVA_OPTS: "-Xmx7500m -Xms4000m"
             LOAD_FROM_DATA: "${LOAD_FROM_DATA}"
+            DEPLOY_PATH: "${DEPLOY_PATH}"
+            DHIS2_AUTH: "${DHIS2_AUTH}"
         entrypoint: sh /config/dhis2-core-entrypoint.sh
         command: sh /config/dhis2-core-start.sh
         depends_on:

src/d2_docker/utils.py

@@ -152,7 +152,7 @@ def get_image_status(image_name, first_port=8080):
         if len(parts) != 3:
             continue
         image_name_part, container_name, ports = parts
-        indexed_service = container_name.split("_")[-2:]
+        indexed_service = container_name.split("_", 2)[-2:]
         if image_name_part == final_image_name and indexed_service:
             service = indexed_service[0]
             containers[service] = container_name
@@ -201,6 +201,9 @@ def run_docker_compose(
     load_from_data=True,
     post_sql_dir=None,
     scripts_dir=None,
+    deploy_path=None,
+    dhis2_auth=None,
+    tomcat_server=None,
     **kwargs,
 ):
     """
@@ -224,15 +227,18 @@ def run_docker_compose(
         # Set default values for directory, required by docker-compose volumes section
         ("POST_SQL_DIR", post_sql_dir_abs),
         ("SCRIPTS_DIR", scripts_dir_abs),
+        ("DEPLOY_PATH", deploy_path or ""),
+        ("DHIS2_AUTH", dhis2_auth or ""),
+        ("TOMCAT_SERVER", get_abs_file_for_docker_volume(tomcat_server)),
     ]
-    env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v)
+    env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None)
 
     yaml_path = os.path.join(os.path.dirname(__file__), "docker-compose.yml")
     return run(["docker-compose", "-f", yaml_path, "-p", project_name, *args], env=env, **kwargs)
 
 
 def get_absdir_for_docker_volume(directory):
-    """Return absolute path for given directory, with default fallback."""
+    """Return absolute path for given directory, with fallback to empty directory."""
     if not directory:
         empty_directory = os.path.join(os.path.dirname(__file__), ".empty")
         return empty_directory
@@ -242,6 +248,14 @@ def get_absdir_for_docker_volume(directory):
         return os.path.abspath(directory)
 
 
+def get_abs_file_for_docker_volume(file_path):
+    """Return absolute path for given file, with fallback to empty file."""
+    if not file_path:
+        return os.path.join(os.path.dirname(__file__), ".empty", "placeholder")
+    else:
+        return os.path.abspath(file_path)
+
+
 def get_item_type(name):
     """
     Return "docker-image" if name matches the pattern 'ORG/{DHIS2_DATA_IMAGE}:TAG',