From 1e8922b119d795e9ea5fd4f976ea9c334340d60e Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Wed, 21 Aug 2024 17:13:23 +0200 Subject: [PATCH] switch to docker image for downloader. --- docker-compose.yml | 5 +- wis2downloader/Dockerfile | 67 --------------------------- wis2downloader/clean_downloads.cron | 1 - wis2downloader/clean_downloads.py | 40 ---------------- wis2downloader/config/config.template | 18 ------- wis2downloader/entrypoint.sh | 32 ------------- 6 files changed, 2 insertions(+), 161 deletions(-) delete mode 100644 wis2downloader/Dockerfile delete mode 100644 wis2downloader/clean_downloads.cron delete mode 100644 wis2downloader/clean_downloads.py delete mode 100644 wis2downloader/config/config.template delete mode 100644 wis2downloader/entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index 82c21b67e..8e580326b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,9 +146,8 @@ services: wis2downloader: container_name: wis2downloader - build: - context: ./wis2downloader - dockerfile: Dockerfile + image: ghcr.io/wmo-im/wis2downloader:latest + restart: always env_file: - wis2box.env user: ${DOWNLOADER_UID}:${DOCKER_GID} diff --git a/wis2downloader/Dockerfile b/wis2downloader/Dockerfile deleted file mode 100644 index b483f452b..000000000 --- a/wis2downloader/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -FROM python:3.12-slim-bookworm - -SHELL ["/bin/bash", "-c"] - -# default ENV / config -ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr" -ENV DOWNLOAD_BROKER_PORT 443 -ENV DOWNLOAD_BROKER_USERNAME "everyone" -ENV DOWNLOAD_BROKER_PASSWORD "everyone" -ENV DOWNLOAD_BROKER_TRANSPORT "websockets" -ENV DOWNLOAD_DIR "/app/downloads" -ENV DOWNLOAD_MIN_FREE_SPACE_GB 1 -ENV DOWNLOAD_RETENTION_PERIOD_HOURS 24 -ENV DOWNLOAD_VALIDATE_TOPICS "false" -ENV DOWNLOAD_WORKERS 8 -ENV LOG_PATH "/home/wis2downloader/app/logs" -ENV WIS2DOWNLOADER_CONFIG "/home/wis2downloader/app/config/config.json" -ENV USER_ID 12135 - -# Update, upgrade packages and install / clean up -RUN apt-get update && \ - apt-get upgrade && \ - apt-get install -y gettext-base=0.21-12 curl=7.88.1-10+deb12u6 cron=3.0pl1-162 git=1:2.39.2-1.1 && \ - rm -rf /var/lib/apt/lists/* - -# Now setup python env and default user -RUN useradd -l -u "$USER_ID" wis2downloader - -USER wis2downloader -WORKDIR /home/wis2downloader - -USER wis2downloader - -RUN python3.12 -m venv /home/wis2downloader/.venv && \ - echo "source /home/wis2downloader/.venv/bin/activate" >> .bashrc && \ - echo "" >> .bashrc - -# install python dependencies -RUN source /home/wis2downloader/.venv/bin/activate && \ - python -m pip install --no-cache-dir gunicorn==23.0.0 requests==2.32.3 && \ - python -m pip install --no-cache-dir pyopenssl==24.2.1 --upgrade && \ - python -m pip install --no-cache-dir wis2downloader==0.3.0 # git+https://github.com/wmo-im/wis2downloader@docker2 - -# copy config and entrypoint to the Docker image -COPY config/. /home/wis2downloader/app/config -COPY entrypoint.sh /home/wis2downloader/app/entrypoint.sh -COPY clean_downloads.cron /home/wis2downloader/app/clean_downloads.cron -COPY clean_downloads.py /home/wis2downloader/app/clean_downloads.py - -USER root -RUN chown -R wis2downloader /home/wis2downloader/app && \ - chmod +x /home/wis2downloader/app/entrypoint.sh && \ - chmod 600 /home/wis2downloader/app/clean_downloads.py && \ - chmod 600 /home/wis2downloader/app/clean_downloads.cron - -USER wis2downloader -# Set the working directory to /app -WORKDIR /home/wis2downloader -RUN crontab ./app/clean_downloads.cron - -# Add healthcheck -HEALTHCHECK --interval=1m --timeout=3s \ - CMD curl -f http://localhost:5000/subscriptions || exit 1 - -ENTRYPOINT [ "/home/wis2downloader/app/entrypoint.sh" ] -# Run wis2downloader when the container launches -CMD ["/bin/bash", "-c", "gunicorn --bind 0.0.0.0:5000 --workers 1 wis2downloader.app:app"] \ No newline at end of file diff --git a/wis2downloader/clean_downloads.cron b/wis2downloader/clean_downloads.cron deleted file mode 100644 index 34ed209df..000000000 --- a/wis2downloader/clean_downloads.cron +++ /dev/null @@ -1 +0,0 @@ -*/10 * * * * source /home/wis2downloader/.venv/bin/activate && python /home/wis2downloader/app/clean_downloads.py > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/wis2downloader/clean_downloads.py b/wis2downloader/clean_downloads.py deleted file mode 100644 index d26dc1c70..000000000 --- a/wis2downloader/clean_downloads.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import time - -# get download dir from environment variable -download_dir = os.environ.get('DOWNLOAD_DIR', '/home/wis2downloader/app/data/downloads') # noqa -# get retention period from environment variable -retention_period_hours = int(os.environ.get('DOWNLOAD_RETENTION_PERIOD_HOURS', None)) # noqa - - -def clean_directory(directory): - # get the current time - current_time = time.time() - - files_removed = 0 - directories_removed = 0 - # loop through the files in the directory, including subdirectories - for file in os.listdir(directory): - # get the full path of the file - file_path = os.path.join(directory, file) - # check if the path is a file or a directory - if os.path.isfile(file_path): - # get the time the file was last modified - file_time = os.path.getmtime(file_path) - # check if the file is older than the retention period - if current_time - file_time > retention_period_hours * 3600: - os.remove(file_path) - files_removed += 1 - elif os.path.isdir(file_path): - # recursively clean the directory - clean_directory(file_path) - # if the directory is empty, remove it - if not os.listdir(file_path): - os.rmdir(file_path) - directories_removed += 1 - print(f'CLEANER: removed {files_removed} old files and {directories_removed} empty directories') # noqa - - -# start cleaning from the download directory -if retention_period_hours is not None: - clean_directory(download_dir) diff --git a/wis2downloader/config/config.template b/wis2downloader/config/config.template deleted file mode 100644 index 8a213c7fb..000000000 --- a/wis2downloader/config/config.template +++ /dev/null @@ -1,18 +0,0 @@ -{ - "broker_hostname": "${DOWNLOAD_BROKER_HOST}", - "broker_port": ${DOWNLOAD_BROKER_PORT}, - "broker_username": "${DOWNLOAD_BROKER_USERNAME}", - "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", - "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", - "download_workers": ${DOWNLOAD_WORKERS}, - "min_free_space": ${DOWNLOAD_MIN_FREE_SPACE_GB}, - "validate_topics": ${DOWNLOAD_VALIDATE_TOPICS}, - "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json", - "download_dir": "${DOWNLOAD_DIR}", - "log_level": "INFO", - "base_url": "http://localhost:5000", - "flask_host": "0.0.0.0", - "flask_port": 5000, - "save_logs": false, - "log_path": "logs" -} \ No newline at end of file diff --git a/wis2downloader/entrypoint.sh b/wis2downloader/entrypoint.sh deleted file mode 100644 index cdef573b4..000000000 --- a/wis2downloader/entrypoint.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -echo "Download directory in container: $DOWNLOAD_DIR" - -# ensure DOWNLOAD_DIR exists -if [ ! -d "$DOWNLOAD_DIR" ]; then - echo "Creating download directory: $DOWNLOAD_DIR" - mkdir -p "$DOWNLOAD_DIR" -fi - -envsubst < /home/wis2downloader/app/config/config.template > /home/wis2downloader/app/config/config.json - -# if session-info.json does not exists in $DOWNLOAD_DIR, create it -if [ ! -f "$DOWNLOAD_DIR/.session-info.json" ]; then - echo "Creating .session-info.json" - echo "{" > "$DOWNLOAD_DIR/.session-info.json" - echo ' "topics": {},' >> "$DOWNLOAD_DIR/.session-info.json" - # generate a random string for client_id - echo "Generating random client_id" - client_id=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1) - echo " \"client_id\": \"wis2box-wis2downloader-${client_id}\"" >> "$DOWNLOAD_DIR/.session-info.json" - echo "}" >> "$DOWNLOAD_DIR/.session-info.json" -fi - -# print the config -echo "Config:" -cat /home/wis2downloader/app/config/config.json -echo "Initial session info:" -cat "$DOWNLOAD_DIR/.session-info.json" -# activate python env -# shellcheck source=/dev/null -source /home/wis2downloader/.venv/bin/activate -exec "$@" \ No newline at end of file