From cd2ee3d21cfacc91840d33179652b56d3579ffac Mon Sep 17 00:00:00 2001 From: jpantos <150924733+jpantos@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:59:08 +0200 Subject: [PATCH] feat: PAN-2112 development watch mode (#100) * feat: PAN-2112 development watch mode --- .github/workflows/ci.yaml | 1 + Makefile | 33 +++++++--- README.md | 18 ++++++ debian/rules | 3 +- docker-compose.override.yml | 14 ++++- docker-compose.yml | 76 ++++++++++++++++++++++++ linux/scripts/pantos-service-node-celery | 12 +++- linux/scripts/start-web.py | 7 +++ 8 files changed, 152 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ee40ffb..6763571 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -336,6 +336,7 @@ jobs: sparse-checkout: | docker-compose.yml docker-compose.override.yml + service-node-config.docker.env Makefile - name: Download artifacts diff --git a/Makefile b/Makefile index 50e2096..6e971f7 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ PYTHON_FILES_WITHOUT_TESTS := pantos/servicenode linux/scripts/start-web.py PYTHON_FILES := $(PYTHON_FILES_WITHOUT_TESTS) tests STACK_BASE_NAME=stack-service-node INSTANCE_COUNT ?= 1 +DEV_MODE ?= false .PHONY: check-version check-version: @@ -246,16 +247,34 @@ docker-build: docker buildx bake -f docker-compose.yml --load $(ARGS); \ fi +.PHONY: docker docker: check-swarm-init docker-build @for i in $$(seq 1 $(INSTANCE_COUNT)); do \ - ( \ - STACK_NAME="${STACK_BASE_NAME}-${STACK_IDENTIFIER}-$$i"; \ - export INSTANCE=$$i; \ - echo "Deploying stack $$STACK_NAME"; \ - docker compose -f docker-compose.yml -f docker-compose.override.yml -p $$STACK_NAME $(EXTRA_COMPOSE) up -d --wait $(ARGS); \ - ) & \ + ( \ + export STACK_NAME="${STACK_BASE_NAME}-${STACK_IDENTIFIER}-$$i"; \ + export INSTANCE=$$i; \ + echo "Deploying stack $$STACK_NAME"; \ + if [ "$(DEV_MODE)" = "true" ]; then \ + echo "Running in development mode"; \ + export ARGS="$(ARGS) --watch"; \ + docker compose -f docker-compose.yml -f docker-compose.override.yml -p $$STACK_NAME $$EXTRA_COMPOSE up $$ARGS & \ + COMPOSE_PID=$$!; \ + trap 'echo "Caught SIGINT, killing background processes..."; kill $$COMPOSE_PID; exit 1' SIGINT; \ + else \ + export ARGS="--detach --wait $(ARGS)"; \ + docker compose -f docker-compose.yml -f docker-compose.override.yml -p $$STACK_NAME $$EXTRA_COMPOSE up $$ARGS; \ + fi; \ + trap 'exit 1' SIGINT; \ + echo "Stack $$STACK_NAME deployed"; \ + if [ "$(DEV_MODE)" = "true" ]; then \ + wait $$COMPOSE_PID; \ + fi; \ + ) & \ done; \ - wait + trap 'echo "Caught SIGINT, killing all background processes..."; kill 0; exit 1' SIGINT; \ + wait + # We need to use compose because swarm goes absolutely crazy on MacOS when using cross architecture + # And can't pull the correct images #docker stack deploy -c docker-compose.yml -c docker-compose.override.yml $$STACK_NAME --with-registry-auth --detach=false $(ARGS) & \ .PHONY: docker-remove diff --git a/README.md b/README.md index 0e8a996..37c8068 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,24 @@ You can run a local setup with docker by doing the following steps: - Ensure you have a `signer_key` file located in the same directory. If you don't, you can create one with `make signer-key` - Run `make docker` +##### Local development with Docker + +You can do local development with Docker by enabling dev mode (Docker watch mode). To do so, set the environment variable `DEV_MODE` to true, like this: + +`DEV_MODE=true make docker` + +#### Multiple local deployments + +We support multiple local deployments, for example for testing purposes, you can run the stacks like this: + +`make docker INSTANCE_COUNT=` + +To remove all the stacks, run the following: + +`make docker-remove` + +Please note that this mode uses an incremental amount of resources and that Docker Desktop doesn't fully support displaying it, but it should be good enough to test multiple service nodes locally. + ##### Production Setup The production setup is slightly different, for convenience we provide a separate `.env` file and `make` method. diff --git a/debian/rules b/debian/rules index 3512a84..aa4a393 100755 --- a/debian/rules +++ b/debian/rules @@ -14,7 +14,7 @@ DH_VERBOSE=1 export DH_VIRTUALENV_INSTALL_ROOT=/opt/pantos export POETRY_VIRTUALENVS_CREATE=false -PYTHON_VERSION ?= 3.12 +PYTHON_VERSION = 3.12.4 SNAKE=debian/$(PACKAGE)$(DH_VIRTUALENV_INSTALL_ROOT)/$(PACKAGE)/bin/python3 EXTRA_REQUIREMENTS=--upgrade-pip --preinstall "setuptools>=38" --preinstall "poetry" --preinstall "dh-poetry" DH_VENV_ARGS=--builtin-venv --python=$(SNAKE) $(EXTRA_REQUIREMENTS) \ @@ -44,6 +44,7 @@ build-arch: override_dh_virtualenv: . $$(conda info --base)/etc/profile.d/conda.sh && \ + conda search --full-name python && \ conda create -y -c defaults -c conda-forge --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \ conda activate $(POETRY_VIRTUALENVS_PATH) && \ dh_virtualenv $(DH_VENV_ARGS) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 6c66d1b..adb8c5d 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -9,7 +9,7 @@ services: networks: pantos-service-node: pantos-ethereum: - entrypoint: sh -c 'set -a; . /etc/pantos/eth-data/ETHEREUM.env && . /etc/pantos/bnb-data/BNB.env; set +a; exec /usr/bin/pantos-service-node-server' + entrypoint: sh -c 'set -a; . /etc/pantos/eth-data/ETHEREUM.env && . /etc/pantos/bnb-data/BNB_CHAIN.env; set +a; exec /usr/bin/pantos-service-node-server' environment: PANTOS_ENV_FILE: /etc/pantos/service-node-config.env APP_URL: http://localhost:808${INSTANCE-1} @@ -32,12 +32,17 @@ services: source: ./service-node-config.docker.env target: /etc/pantos/service-node-config.env read_only: true + develop: + watch: + - action: sync+restart + path: service-node-config.docker.env + target: /root/service-node-config.docker.env worker: networks: pantos-service-node: pantos-ethereum: - entrypoint: sh -c 'set -a; . /etc/pantos/eth-data/ETHEREUM.env && . /etc/pantos/bnb-data/BNB.env; set +a; exec /usr/bin/pantos-service-node-celery' + entrypoint: sh -c 'set -a; . /etc/pantos/eth-data/ETHEREUM.env && . /etc/pantos/bnb-data/BNB_CHAIN.env; set +a; exec /usr/bin/pantos-service-node-celery' environment: PANTOS_ENV_FILE: /etc/pantos/service-node-config.env APP_URL: http://localhost:808${INSTANCE-1} @@ -63,6 +68,11 @@ services: source: ./service-node-config.docker.env target: /etc/pantos/service-node-config.env read_only: true + develop: + watch: + - action: sync+restart + path: service-node-config.docker.env + target: /root/service-node-config.docker.env volumes: bnb-data: diff --git a/docker-compose.yml b/docker-compose.yml index 2a34c57..367861b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,7 @@ services: DB_URL: postgresql://pantos-service-node:pantos@db/pantos-service-node CELERY_BROKER: amqp://pantos-service-node:pantos@broker:5672/pantos-service-node CELERY_BACKEND: db+postgresql://pantos-service-node:pantos@db/pantos-service-node-celery + DEV_MODE: ${DEV_MODE} ports: - 808${INSTANCE-0}:8080 healthcheck: @@ -47,6 +48,42 @@ services: condition: service_healthy db: condition: service_healthy + develop: + watch: + - action: sync + path: ./pantos + target: /opt/pantos/pantos-service-node/lib/python3.12/site-packages/pantos + - action: sync+restart + path: ./linux/scripts/start-web.py + target: /opt/pantos/pantos-service-node/bin/start-web.py + - action: sync+restart + path: ./linux/scripts/pantos-service-node-server + target: /opt/pantos/pantos-service-node/bin/pantos-service-node-server + - action: sync+restart + path: service-node-config.yml + target: /opt/pantos/pantos-service-node/service-node-config.yml + - action: sync+restart + path: alembic.ini + target: /opt/pantos/pantos-service-node/alembic.ini + - action: sync+restart + path: service-node-config.env + target: /root/service-node-config.env + - action: sync+restart + path: bids.yml + target: /etc/service-node-bids.yml + - action: rebuild + path: Dockerfile + - action: rebuild + path: debian + - action: rebuild + path: configurator + - action: rebuild + path: linux/etc + - action: rebuild + path: pyproject.toml + - action: rebuild + path: poetry.lock + worker: deploy: restart_policy: @@ -77,6 +114,7 @@ services: DB_URL: postgresql://pantos-service-node:pantos@db/pantos-service-node CELERY_BROKER: amqp://pantos-service-node:pantos@broker:5672/pantos-service-node CELERY_BACKEND: db+postgresql://pantos-service-node:pantos@db/pantos-service-node-celery + DEV_MODE: ${DEV_MODE} healthcheck: test: ["CMD", "/usr/bin/pantos-service-node-celery", "--status"] interval: 5s @@ -87,6 +125,39 @@ services: # Wait for the app to setup the DB app: condition: service_healthy + develop: + watch: + - action: sync + path: ./pantos + target: /opt/pantos/pantos-service-node/lib/python3.12/site-packages/pantos + - action: sync+restart + path: ./linux/scripts/pantos-service-node-celery + target: /opt/pantos/pantos-service-node/bin/pantos-service-node-celery + - action: sync+restart + path: service-node-config.yml + target: /opt/pantos/pantos-service-node/service-node-config.yml + - action: sync+restart + path: alembic.ini + target: /opt/pantos/pantos-service-node/alembic.ini + - action: sync+restart + path: service-node-config.env + target: /root/service-node-config.env + - action: sync+restart + path: bids.yml + target: /etc/service-node-bids.yml + - action: rebuild + path: Dockerfile + - action: rebuild + path: debian + - action: rebuild + path: configurator + - action: rebuild + path: linux/etc + - action: rebuild + path: pyproject.toml + - action: rebuild + path: poetry.lock + db: image: postgres:latest deploy: @@ -112,6 +183,11 @@ services: source: ./db_init.sh target: /docker-entrypoint-initdb.d/init-user-db.sh read_only: true + develop: + watch: + - action: rebuild + path: db_init.sh + broker: image: rabbitmq:latest deploy: diff --git a/linux/scripts/pantos-service-node-celery b/linux/scripts/pantos-service-node-celery index 626c430..18e92dd 100644 --- a/linux/scripts/pantos-service-node-celery +++ b/linux/scripts/pantos-service-node-celery @@ -49,9 +49,16 @@ while [ $# -gt 0 ]; do esac done +PROGRAM="./bin/python" +if [ "$DEV_MODE" = "true" ]; then + echo "Running in development mode" + ./bin/pip install watchdog[watchmedo] + PROGRAM="./bin/watchme autorestart --directory=$APP_DIR --pattern=*.py --recursive -- ./bin/python" +fi + if [ -n "$PANTOS_STATUS_MONITOR" ]; then echo "Starting the status monitor" - exec ./bin/python -m celery -A pantos.servicenode flower & + $PROGRAM -m celery -A pantos.servicenode flower & fi # Check if the current user is not "pantos-service-node" and if so, switch to that user @@ -61,9 +68,10 @@ if [ "$(id -u)" -ne "$(id -u "$APP_NAME")" ]; then EXTRA_ARGS="--uid $(id -u "$APP_NAME")" fi + while true; do echo "Starting the celery worker" - ./bin/python -m celery -A pantos.servicenode worker $EXTRA_ARGS -l INFO -n pantos.servicenode -Q transfers,bids + $PROGRAM -m celery -A pantos.servicenode worker $EXTRA_ARGS -l INFO -n pantos.servicenode -Q transfers,bids PYTHON_EXIT_CODE=$? if [ "$PANTOS_CELERY_AUTORESTART" != "true" ]; then diff --git a/linux/scripts/start-web.py b/linux/scripts/start-web.py index ad81722..58a3e41 100644 --- a/linux/scripts/start-web.py +++ b/linux/scripts/start-web.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Start the service node server.""" +import os import subprocess import sys from importlib import resources @@ -80,5 +81,11 @@ server_run_command = ['runuser', '-u', USER_NAME, '--' ] + gunicorn_command.split() +if os.getenv('DEV_MODE', False) == 'true': + print('Running in development mode') + server_run_command = server_run_command + [ + '--reload', '--log-level', 'debug' + ] + print('Starting the server...') subprocess.run(server_run_command, check=True, text=True)