diff --git a/README.md b/README.md index 905e658..8d46a35 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ $ d2-docker create core eyeseetea/dhis2-core:2.30 --war=dhis.war --dhis2-home=/t ### Create a base DHIS2 data image -Create a dhis2-data image from a .sql.gz SQL file and the apps directory to include: +Create a dhis2-data image from a .sql.gz SQL file and the apps and documents directory to include: ``` -$ d2-docker create data eyeseetea/dhis2-data:2.30-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps] +$ d2-docker create data eyeseetea/dhis2-data:2.30-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps] [--documents-dir=path/to/document] ``` ### Start a DHIS2 instance @@ -82,6 +82,15 @@ Some notes: - Use option `-auth` to pass the instance authentication (`USER:PASS`). It will be used to call post-tomcat scripts. - Use option `--run-sql=DIRECTORY` to run SQL files (.sql, .sql.gz or .dump files) after the DB has been initialized. - Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to. +- Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`) + +#### Custom DHIS2 dhis.conf + +Copy the default [dhis.conf](https://github.com/EyeSeeTea/d2-docker/blob/master/src/d2_docker/config/DHIS2_home/dhis.conf) and use it as a template to create your own configuration. Then pass it to the `start` command: + +``` +$ d2-docker start --dhis-conf=dhis.conf ... +``` #### Custom Tomcat server.xml @@ -125,7 +134,7 @@ _If only one d2-docker container is active, you can omit the image name._ ### Commit & push an image -This will update the image from the current container (SQL dump and apps): +This will update the image from the current container (SQL dump, apps and documents): ``` $ d2-docker commit @@ -193,7 +202,7 @@ $ d2-docker rm eyeseetea/dhis2-data:2.30-sierra ### Copy Docker images to/from local directories -You can use a Docker image or a data directory (db + apps) as source, that will create a new Docker image _eyeseetea/dhis2-data:2.30-sierra2_ and a `sierra-data/` directory: +You can use a Docker image or a data directory (db + apps + documents) as source, that will create a new Docker image _eyeseetea/dhis2-data:2.30-sierra2_ and a `sierra-data/` directory: ``` $ d2-docker copy eyeseetea/dhis2-data:2.30-sierra eyeseetea/dhis2-data:2.30-sierra2 sierra-data @@ -202,10 +211,10 @@ $ docker image ls | grep 2.30-sierra2 eyeseetea/dhis2-data 2.30-sierra2 930aced0d915 1 minutes ago 106MB $ ls sierra-data/ -apps db.sql.gz +apps document db.sql.gz ``` -Alternatively, you can use a data directory (db + apps) as source and create Docker images from it: +Alternatively, you can use a data directory (db + apps + documents) as source and create Docker images from it: ``` $ d2-docker copy sierra-data eyeseetea/dhis2-data:2.30-sierra3 eyeseetea/dhis2-data:2.30-sierra4 diff --git a/setup.py b/setup.py index ab8f88f..8ff1329 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setuptools.setup( name="d2_docker", - version="1.2.0", + version="1.3.0", description="Dockers for DHIS2 instances", long_description=open("README.md", encoding="utf-8").read(), keywords=["python"], diff --git a/src/d2_docker/commands/copy.py b/src/d2_docker/commands/copy.py index 293582d..f35b8fe 100644 --- a/src/d2_docker/commands/copy.py +++ b/src/d2_docker/commands/copy.py @@ -40,3 +40,5 @@ def copy(source, destinations, docker_dir): utils.copytree(source, dest) else: raise utils.D2DockerError("Not implemented") + + logger.info("Done") diff --git a/src/d2_docker/commands/create.py b/src/d2_docker/commands/create.py index 94e2504..b61364c 100644 --- a/src/d2_docker/commands/create.py +++ b/src/d2_docker/commands/create.py @@ -21,6 +21,7 @@ def setup(parser): data_parser.add_argument("data_image", metavar="IMAGE", help="Image core name") data_parser.add_argument("--sql", help=".sql (plain text), .sql.gz (gzipped plain text format) or .dump database file (binary format)") data_parser.add_argument("--apps-dir", help="Directory containing Dhis2 apps") + data_parser.add_argument("--documents-dir", help="Directory containing Dhis2 documents") def run(args): @@ -55,6 +56,10 @@ def create_data(args): dest_apps_dir = os.path.join(build_dir, "apps") utils.logger.debug("Copy apps: {} -> {}".format(args.apps_dir, dest_apps_dir)) utils.copytree(args.apps_dir, dest_apps_dir) + if args.documents_dir: + dest_documents_dir = os.path.join(build_dir, "document") + utils.logger.debug("Copy documents: {} -> {}".format(args.documents_dir, dest_documents_dir)) + utils.copytree(args.documents_dir, dest_documents_dir) if args.sql: utils.logger.debug("Copy DB file: {} -> {}".format(args.sql, db_path)) shutil.copy(args.sql, db_path) diff --git a/src/d2_docker/commands/start.py b/src/d2_docker/commands/start.py index f4b37ec..bc4c882 100644 --- a/src/d2_docker/commands/start.py +++ b/src/d2_docker/commands/start.py @@ -9,8 +9,10 @@ def setup(parser): d2_docker_path = os.path.abspath(d2_docker.__path__[0]) - server_xml_path = os.path.join(d2_docker_path, "config", "server.xml") + server_xml_path = utils.get_config_file("server.xml") server_xml_help = "Use a custom Tomcat server.xml file. Template: {0}".format(server_xml_path) + dhis_conf_path = utils.get_config_file("DHIS2_home/dhis.conf") + dhis_conf_help = "Use a custom dhis.conf file. Template: {0}".format(dhis_conf_path) parser.add_argument( "image_or_file", metavar="IMAGE_OR_EXPORT_FILE", help="Docker image or exported file" @@ -24,6 +26,7 @@ def setup(parser): "-k", "--keep-containers", action="store_true", help="Keep existing containers" ) parser.add_argument("--tomcat-server-xml", metavar="FILE", help=server_xml_help) + parser.add_argument("--dhis-conf", metavar="FILE", help=dhis_conf_help) parser.add_argument("--run-sql", metavar="DIRECTORY", help="Run .sql[.gz] files in directory") parser.add_argument( "--run-scripts", @@ -33,6 +36,7 @@ def setup(parser): parser.add_argument("--pull", action="store_true", help="Force a pull from docker hub") parser.add_argument("-p", "--port", type=int, metavar="N", help="Set Dhis2 instance port") parser.add_argument("--deploy-path", type=str, help="Set Tomcat context.path") + parser.add_argument("--java-opts", type=str, help="Set Tomcat JAVA_OPTS") def run(args): @@ -81,7 +85,7 @@ def start(args, image_name): bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None] ) - deploy_path = ("/" + re.sub("^/*", "", args.deploy_path) if args.deploy_path else "") + deploy_path = "/" + re.sub("^/*", "", args.deploy_path) if args.deploy_path else "" with utils.stop_docker_on_interrupt(image_name, core_image): utils.run_docker_compose( @@ -94,7 +98,9 @@ def start(args, image_name): scripts_dir=args.run_scripts, deploy_path=deploy_path, dhis2_auth=args.auth, - tomcat_server=args.tomcat_server_xml + tomcat_server=args.tomcat_server_xml, + dhis_conf=args.dhis_conf, + java_opts=args.java_opts, ) if args.detach: diff --git a/src/d2_docker/config/dhis2-core-start.sh b/src/d2_docker/config/dhis2-core-start.sh index 989277c..c24349d 100755 --- a/src/d2_docker/config/dhis2-core-start.sh +++ b/src/d2_docker/config/dhis2-core-start.sh @@ -25,7 +25,8 @@ scripts_dir="/data/scripts" root_db_path="/data/db" post_db_path="/data/db/post" source_apps_path="/data/apps" -dest_apps_path="/DHIS2_home/files/" +source_documents_path="/data/document" +files_path="/DHIS2_home/files/" tomcat_conf_dir="/usr/local/tomcat/conf" debug() { @@ -34,6 +35,8 @@ debug() { run_sql_files() { base_db_path=$(test "${LOAD_FROM_DATA}" = "yes" && echo "$root_db_path" || echo "$post_db_path") + debug "Files in data path" + find "$base_db_path" >&2; find "$base_db_path" -type f \( -name '*.dump' \) | sort | while read -r path; do @@ -69,10 +72,18 @@ run_post_scripts() { } copy_apps() { - debug "Copy Dhis2 apps: $source_apps_path -> $dest_apps_path" - mkdir -p "$dest_apps_path/apps" + debug "Copy Dhis2 apps: $source_apps_path -> $files_path" + mkdir -p "$files_path/apps" if test -e "$source_apps_path"; then - cp -Rv "$source_apps_path" "$dest_apps_path" + cp -Rv "$source_apps_path" "$files_path" + fi +} + +copy_documents() { + debug "Copy Dhis2 documents: $source_documents_path -> $files_path" + mkdir -p "$files_path/document" + if test -e "$source_documents_path"; then + cp -Rv "$source_documents_path" "$files_path" fi } @@ -107,6 +118,7 @@ run() { local host=$1 psql_port=$2 setup_tomcat copy_apps + copy_documents wait_for_postgres run_sql_files || true run_pre_scripts || true diff --git a/src/d2_docker/docker-compose.yml b/src/d2_docker/docker-compose.yml index a87ce84..d203f64 100644 --- a/src/d2_docker/docker-compose.yml +++ b/src/d2_docker/docker-compose.yml @@ -6,6 +6,7 @@ services: - "com.eyeseetea.image-name=${DHIS2_DATA_IMAGE}" volumes: - home:/DHIS2_home + - ${DHIS_CONF}:/config/DHIS2_home/dhis.conf - ./config:/config - data:/data - "${TOMCAT_SERVER}:/config/override/tomcat/server.xml" @@ -13,7 +14,7 @@ services: - "${POST_SQL_DIR}:/data/db/post" environment: CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH}" - JAVA_OPTS: "-Xmx7500m -Xms4000m" + JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}" LOAD_FROM_DATA: "${LOAD_FROM_DATA}" DEPLOY_PATH: "${DEPLOY_PATH}" DHIS2_AUTH: "${DHIS2_AUTH}" diff --git a/src/d2_docker/images/dhis2-data/Dockerfile b/src/d2_docker/images/dhis2-data/Dockerfile index 093e244..d18cb84 100644 --- a/src/d2_docker/images/dhis2-data/Dockerfile +++ b/src/d2_docker/images/dhis2-data/Dockerfile @@ -6,5 +6,6 @@ COPY db/ /data/db/ COPY run.sh /usr/local/bin/ COPY apps/ /data/apps +COPY document/ /data/document CMD ["sh", "/usr/local/bin/run.sh"] diff --git a/src/d2_docker/images/dhis2-data/document/.placeholder b/src/d2_docker/images/dhis2-data/document/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/src/d2_docker/utils.py b/src/d2_docker/utils.py index 35a8835..cbd4844 100644 --- a/src/d2_docker/utils.py +++ b/src/d2_docker/utils.py @@ -11,6 +11,7 @@ from distutils import dir_util from pathlib import Path +import d2_docker from .image_name import ImageName PROJECT_NAME_PREFIX = "d2-docker" @@ -63,6 +64,14 @@ def run(command_parts, raise_on_error=True, env=None, capture_output=False, **kw raise D2DockerError(msg.format(cmd, exc.returncode, exc.stderr)) +@contextlib.contextmanager +def possible_errors(): + try: + yield + except D2DockerError as exc: + pass + + def get_free_port(start=8080, end=65535): for port in range(start, end): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: @@ -202,6 +211,8 @@ def run_docker_compose( post_sql_dir=None, scripts_dir=None, deploy_path=None, + dhis_conf=None, + java_opts=None, dhis2_auth=None, tomcat_server=None, **kwargs, @@ -228,8 +239,10 @@ def run_docker_compose( ("POST_SQL_DIR", post_sql_dir_abs), ("SCRIPTS_DIR", scripts_dir_abs), ("DEPLOY_PATH", deploy_path or ""), + ("JAVA_OPTS", java_opts or ""), ("DHIS2_AUTH", dhis2_auth or ""), - ("TOMCAT_SERVER", get_abs_file_for_docker_volume(tomcat_server)), + ("TOMCAT_SERVER", get_absfile_for_docker_volume(tomcat_server)), + ("DHIS_CONF", get_config_path("DHIS2_home/dhis.conf", dhis_conf)), ] env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None) @@ -237,6 +250,10 @@ def run_docker_compose( return run(["docker-compose", "-f", yaml_path, "-p", project_name, *args], env=env, **kwargs) +def get_config_path(default_filename, path): + return os.path.abspath(path) if path else get_config_file(default_filename) + + def get_absdir_for_docker_volume(directory): """Return absolute path for given directory, with fallback to empty directory.""" if not directory: @@ -248,7 +265,7 @@ def get_absdir_for_docker_volume(directory): return os.path.abspath(directory) -def get_abs_file_for_docker_volume(file_path): +def get_absfile_for_docker_volume(file_path): """Return absolute path for given file, with fallback to empty file.""" if not file_path: return os.path.join(os.path.dirname(__file__), ".empty", "placeholder") @@ -315,7 +332,7 @@ def copy_image(docker_dir, source_image, dest_image): def build_image_from_directory(docker_dir, data_dir, dest_image_name): - """Build docker image from data (db + apps) directory.""" + """Build docker image from data (db + apps + documents) directory.""" with tempfile.TemporaryDirectory() as temp_dir_root: logger.info("Create temporal directory: {}".format(temp_dir_root)) temp_dir = os.path.join(temp_dir_root, "contents") @@ -329,23 +346,30 @@ def build_image_from_directory(docker_dir, data_dir, dest_image_name): def export_data_from_image(source_image, dest_path): + logger.info("Export data from image: {} -> {}".format(source_image, dest_path)) result = run(["docker", "create", source_image], capture_output=True) container_id = result.stdout.decode("utf8").splitlines()[0] mkdir_p(dest_path) try: run(["docker", "cp", container_id + ":" + "/data/db", dest_path]) - run(["docker", "cp", container_id + ":" + "/data/apps", dest_path]) + with possible_errors(): + run(["docker", "cp", container_id + ":" + "/data/apps", dest_path]) + run(["docker", "cp", container_id + ":" + "/data/document", dest_path]) finally: run(["docker", "rm", "-v", container_id]) def export_data_from_running_containers(image_name, containers, destination): - """Export data (db + apps) from a running Docker container to a destination directory.""" + """Export data (db + apps + documents) from a running Docker container to a destination directory.""" logger.info("Copy Dhis2 apps") - apps_source = "{}:/DHIS2_home/files/apps/".format(containers["core"]) mkdir_p(destination) + + apps_source = "{}:/DHIS2_home/files/apps/".format(containers["core"]) run(["docker", "cp", apps_source, destination]) + documents_source = "{}:/DHIS2_home/files/document/".format(containers["core"]) + run(["docker", "cp", documents_source, destination]) + db_path = os.path.join(destination, "db", "db.sql.gz") export_database(image_name, db_path) @@ -452,7 +476,12 @@ def wait_for_server(port): def create_core( - *, docker_dir, image, version=None, war=None, dhis2_home_paths=None, + *, + docker_dir, + image, + version=None, + war=None, + dhis2_home_paths=None, ): logger.info("Create core image: {}".format(image)) @@ -478,4 +507,9 @@ def create_core( run(["docker", "build", build_dir, "--tag", image]) +def get_config_file(filename): + d2_docker_path = os.path.abspath(d2_docker.__path__[0]) + return os.path.join(d2_docker_path, "config", filename) + + logger = get_logger()