diff --git a/MANIFEST.in b/MANIFEST.in index 4e8eb5a..35a95ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include requirements.txt include src/d2_docker/docker-compose.yml recursive-include src/d2_docker/config * recursive-include src/d2_docker/images * +recursive-include src/d2_docker/.empty * diff --git a/README.md b/README.md index fa197f0..905e658 100644 --- a/README.md +++ b/README.md @@ -76,10 +76,42 @@ Some notes: - By default, the image `dhis2-core` from the same organisation will be used, keeping the first part of the tag (using `-` as separator). For example: `eyeseetea/dhis2-data:2.30-sierra` will use core `eyeseetea/dhis2-core:2.30`. If you need a custom image to be used, use `--core-image= eyeseetea/dhis2-core:2.30-custom`. - Once started, you can connect to the DHIS2 instance (`http://localhost:PORT`) where _PORT_ is the first available port starting from 8080. You can run many images at the same time, but not the same image more than once. You can specify the port with option `-p PORT`. - Use option `--pull` to overwrite the local images with the images in the hub. -- Use option `--detach` to run containers in the background. -- Use option `-k`/`--keep-containers` to re-use existing containers, so data from the previous run will be kept. +- Use option `--detach` to run the container in the background. +- Use option `--deploy-path` to run the container with a deploy path namespace (i.e: `--deploy-path=dhis2` serves `http://localhost:8080/dhis2`) +- Use option `-k`/`--keep-containers` to re-use existing docker containers, so data from the previous run will be kept. +- Use option `-auth` to pass the instance authentication (`USER:PASS`). It will be used to call post-tomcat scripts. - Use option `--run-sql=DIRECTORY` to run SQL files (.sql, .sql.gz or .dump files) after the DB has been initialized. -- Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080`, regardless of the public port that the instance is exposed to. Of course, this endpoint is only available for post-scripts. +- Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to. + +#### Custom Tomcat server.xml + +Copy the default [server.xml](https://github.com/EyeSeeTea/d2-docker/blob/master/src/d2_docker/config/server.xml) and use it as a template to create your own configuration. Then pass it to the `start` command: + +``` +$ d2-docker start --tomcat-server-xml=server-xml.xml ... +``` + +Note that you should not change the catalina connector port (8080, by default). A typical configuration to use _https_ would look like this: + +``` + + ... + + + ... +/> +``` ### Show logs for running containers @@ -214,11 +246,11 @@ $ d2-docker upgrade \ Migration folder `upgrade-sierra` should then contain data to be used in each intermediate upgrade version. Supported migration data: -- DHIS2 war file: `dhis.war` (if not specified, it will be download from the releases page) -- DHIS2 home files: `dhis2-home/` -- Shell scripts (pre-tomcat): `*.sh` -- Shell scripts (post-tomcat): `post-*.sh` -- SQL files: `*.sql` +- DHIS2 war file: `dhis.war` (if not specified, it will be download from the releases page) +- DHIS2 home files: `dhis2-home/` +- Shell scripts (pre-tomcat): `*.sh` +- Shell scripts (post-tomcat): `post-*.sh` +- SQL files: `*.sql` A full example might look: diff --git a/setup.py b/setup.py index 4b7bc72..eed93e9 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setuptools.setup( name="d2_docker", - version="1.0.6", + version="1.0.8", description="Dockers for DHIS2 instances", long_description=open("README.md", encoding="utf-8").read(), keywords=["python"], diff --git a/src/d2_docker/cli.py b/src/d2_docker/cli.py index 9cc46bb..752318f 100644 --- a/src/d2_docker/cli.py +++ b/src/d2_docker/cli.py @@ -18,6 +18,7 @@ run_sql, create, upgrade, + version ) COMMAND_MODULES = [ @@ -35,6 +36,7 @@ run_sql, create, upgrade, + version, ] def get_parser(): @@ -47,7 +49,7 @@ def get_parser(): parser.add_argument( "--log-level", metavar="NOTSET | DEBUG | INFO | WARNING | ERROR | CRITICAL", - default="DEBUG", + default="INFO", help="Run command with the given log level", ) subparsers = parser.add_subparsers(help="Subcommands", dest="command") diff --git a/src/d2_docker/commands/start.py b/src/d2_docker/commands/start.py index f8e367b..f4b37ec 100644 --- a/src/d2_docker/commands/start.py +++ b/src/d2_docker/commands/start.py @@ -1,22 +1,29 @@ import os import re +import d2_docker from d2_docker import utils DESCRIPTION = "Start a container from an existing dhis2-data Docker image or from an exported file" def setup(parser): + d2_docker_path = os.path.abspath(d2_docker.__path__[0]) + server_xml_path = os.path.join(d2_docker_path, "config", "server.xml") + server_xml_help = "Use a custom Tomcat server.xml file. Template: {0}".format(server_xml_path) + parser.add_argument( "image_or_file", metavar="IMAGE_OR_EXPORT_FILE", help="Docker image or exported file" ) utils.add_core_image_arg(parser) + parser.add_argument("--auth", metavar="USER:PASSWORD", help="Dhis2 instance authentication") parser.add_argument( "-d", "--detach", action="store_true", help="Run containers on the background" ) parser.add_argument( "-k", "--keep-containers", action="store_true", help="Keep existing containers" ) + parser.add_argument("--tomcat-server-xml", metavar="FILE", help=server_xml_help) parser.add_argument("--run-sql", metavar="DIRECTORY", help="Run .sql[.gz] files in directory") parser.add_argument( "--run-scripts", @@ -25,6 +32,7 @@ def setup(parser): ) parser.add_argument("--pull", action="store_true", help="Force a pull from docker hub") parser.add_argument("-p", "--port", type=int, metavar="N", help="Set Dhis2 instance port") + parser.add_argument("--deploy-path", type=str, help="Set Tomcat context.path") def run(args): @@ -73,6 +81,8 @@ def start(args, image_name): bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None] ) + deploy_path = ("/" + re.sub("^/*", "", args.deploy_path) if args.deploy_path else "") + with utils.stop_docker_on_interrupt(image_name, core_image): utils.run_docker_compose( ["up", *up_args], @@ -82,6 +92,9 @@ def start(args, image_name): load_from_data=override_containers, post_sql_dir=args.run_sql, scripts_dir=args.run_scripts, + deploy_path=deploy_path, + dhis2_auth=args.auth, + tomcat_server=args.tomcat_server_xml ) if args.detach: diff --git a/src/d2_docker/commands/version.py b/src/d2_docker/commands/version.py new file mode 100644 index 0000000..d1243b9 --- /dev/null +++ b/src/d2_docker/commands/version.py @@ -0,0 +1,20 @@ +from d2_docker import utils +import pkg_resources # part of setuptools + +DESCRIPTION = "Show version" + + +def setup(parser): + pass + + +def run(args): + utils.run(["docker", "-v"]) + utils.run(["docker-compose", "-v"]) + resource = pkg_resources.require("d2-docker") + if resource: + version = pkg_resources.require("d2-docker")[0].version + print("d2-docker version {}".format(version)) + else: + print("Cannot get d2-docker version") + diff --git a/src/d2_docker/config/dhis2-core-start.sh b/src/d2_docker/config/dhis2-core-start.sh index a448580..989277c 100755 --- a/src/d2_docker/config/dhis2-core-start.sh +++ b/src/d2_docker/config/dhis2-core-start.sh @@ -9,11 +9,14 @@ set -e -u -o pipefail # - Run post-tomcat shell scripts # -# Global LOAD_FROM_DATA="yes" | "no" +# Global: LOAD_FROM_DATA="yes" | "no" +# Global: DEPLOY_PATH=string +# Global: DHIS2_AUTH=string export PGPASSWORD="dhis" -dhis2_url="http://localhost:8080" +dhis2_url="http://localhost:8080/$DEPLOY_PATH" +dhis2_url_with_auth="http://$DHIS2_AUTH@localhost:8080/$DEPLOY_PATH" psql_cmd="psql -v ON_ERROR_STOP=0 --quiet -h db -U dhis dhis2" pgrestore_cmd="pg_restore -h db -U dhis -d dhis2" configdir="/config" @@ -23,6 +26,7 @@ root_db_path="/data/db" post_db_path="/data/db/post" source_apps_path="/data/apps" dest_apps_path="/DHIS2_home/files/" +tomcat_conf_dir="/usr/local/tomcat/conf" debug() { echo "[dhis2-core-start] $*" >&2 @@ -53,14 +57,14 @@ run_sql_files() { run_pre_scripts() { find "$scripts_dir" -type f -name '*.sh' ! \( -name 'post*' \) | sort | while read -r path; do debug "Run pre-tomcat script: $path" - (cd "$(dirname "$path")" && bash "$path") + (cd "$(dirname "$path")" && bash -x "$path") done } run_post_scripts() { find "$scripts_dir" -type f -name '*.sh' -name 'post*' | sort | while read -r path; do debug "Run post-tomcat script: $path" - (cd "$(dirname "$path")" && bash "$path") + (cd "$(dirname "$path")" && bash -x "$path" "$dhis2_url_with_auth") done } @@ -73,9 +77,11 @@ copy_apps() { } setup_tomcat() { + debug "Setup tomcat" cp -v "$configdir/DHIS2_home/dhis.conf" /DHIS2_home/dhis.conf cp -v $homedir/* /DHIS2_home/ || true - cp -v "$configdir/server.xml" /usr/local/tomcat/conf/server.xml + cp -v "$configdir/server.xml" "$tomcat_conf_dir/server.xml" + find "$configdir/override/tomcat/" -maxdepth 1 -type f -size +0 -exec cp -v {} "$tomcat_conf_dir/" \; } wait_for_postgres() { @@ -92,7 +98,7 @@ start_tomcat() { wait_for_tomcat() { debug "Waiting for Tomcat to start: $dhis2_url" - while ! curl -sS -i "$dhis2_url" 2>/dev/null | grep "Location: .*redirect.action"; do + while ! curl -sS -i "$dhis2_url" 2>/dev/null | grep "^Location"; do sleep 1 done } diff --git a/src/d2_docker/config/server.xml b/src/d2_docker/config/server.xml index 86e2951..3e9806e 100644 --- a/src/d2_docker/config/server.xml +++ b/src/d2_docker/config/server.xml @@ -18,14 +18,20 @@ - - + diff --git a/src/d2_docker/docker-compose.yml b/src/d2_docker/docker-compose.yml index c90a685..1a4f18f 100644 --- a/src/d2_docker/docker-compose.yml +++ b/src/d2_docker/docker-compose.yml @@ -8,12 +8,15 @@ services: - home:/DHIS2_home - ./config:/config - data:/data + - "${TOMCAT_SERVER}:/config/override/tomcat/server.xml" - "${SCRIPTS_DIR}:/data/scripts" - "${POST_SQL_DIR}:/data/db/post" environment: - CATALINA_OPTS: "-Dcontext.path=" + CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH}" JAVA_OPTS: "-Xmx7500m -Xms4000m" LOAD_FROM_DATA: "${LOAD_FROM_DATA}" + DEPLOY_PATH: "${DEPLOY_PATH}" + DHIS2_AUTH: "${DHIS2_AUTH}" entrypoint: sh /config/dhis2-core-entrypoint.sh command: sh /config/dhis2-core-start.sh depends_on: diff --git a/src/d2_docker/utils.py b/src/d2_docker/utils.py index 2a2d723..35a8835 100644 --- a/src/d2_docker/utils.py +++ b/src/d2_docker/utils.py @@ -152,7 +152,7 @@ def get_image_status(image_name, first_port=8080): if len(parts) != 3: continue image_name_part, container_name, ports = parts - indexed_service = container_name.split("_")[-2:] + indexed_service = container_name.split("_", 2)[-2:] if image_name_part == final_image_name and indexed_service: service = indexed_service[0] containers[service] = container_name @@ -201,6 +201,9 @@ def run_docker_compose( load_from_data=True, post_sql_dir=None, scripts_dir=None, + deploy_path=None, + dhis2_auth=None, + tomcat_server=None, **kwargs, ): """ @@ -224,15 +227,18 @@ def run_docker_compose( # Set default values for directory, required by docker-compose volumes section ("POST_SQL_DIR", post_sql_dir_abs), ("SCRIPTS_DIR", scripts_dir_abs), + ("DEPLOY_PATH", deploy_path or ""), + ("DHIS2_AUTH", dhis2_auth or ""), + ("TOMCAT_SERVER", get_abs_file_for_docker_volume(tomcat_server)), ] - env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v) + env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None) yaml_path = os.path.join(os.path.dirname(__file__), "docker-compose.yml") return run(["docker-compose", "-f", yaml_path, "-p", project_name, *args], env=env, **kwargs) def get_absdir_for_docker_volume(directory): - """Return absolute path for given directory, with default fallback.""" + """Return absolute path for given directory, with fallback to empty directory.""" if not directory: empty_directory = os.path.join(os.path.dirname(__file__), ".empty") return empty_directory @@ -242,6 +248,14 @@ def get_absdir_for_docker_volume(directory): return os.path.abspath(directory) +def get_abs_file_for_docker_volume(file_path): + """Return absolute path for given file, with fallback to empty file.""" + if not file_path: + return os.path.join(os.path.dirname(__file__), ".empty", "placeholder") + else: + return os.path.abspath(file_path) + + def get_item_type(name): """ Return "docker-image" if name matches the pattern 'ORG/{DHIS2_DATA_IMAGE}:TAG',