From 0f45f18de9ddc47c61e82f3263522ce1c302f23f Mon Sep 17 00:00:00 2001 From: Carlos Wu Fei Date: Sat, 18 Nov 2023 20:31:26 +0100 Subject: [PATCH] Refactor cronjobs as a separate service --- .github/workflows/pr.yml | 29 ++++++++++++++++++ .vscode/launch.json | 18 +++++------ Dockerfile.cronjobs | 9 ++++++ api/cronjobs.py | 65 ++++++++++++++++++++++++++++++++++++++++ api/market_updates.py | 51 ++----------------------------- docker-compose.yml | 14 +++++++++ 6 files changed, 128 insertions(+), 58 deletions(-) create mode 100644 Dockerfile.cronjobs create mode 100644 api/cronjobs.py diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 38386e04e..da07ed430 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -97,6 +97,35 @@ jobs: docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} docker push carloswufei/binbot_streaming + deploy_cronjobs: + name: Deploy cronjobs + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Build image + run: docker build --tag binbot_cronjobs -f Dockerfile.cronjobs . + - name: Test run script + run: | + docker run --network host --name binbot_cronjobs \ + -e MONGO_HOSTNAME=${{ env.MONGO_HOSTNAME }} \ + -e MONGO_PORT=${{ env.MONGO_PORT }} \ + -e MONGO_APP_DATABASE=${{ env.MONGO_APP_DATABASE }} \ + -e MONGO_AUTH_USERNAME=${{ env.MONGO_AUTH_USERNAME }} \ + -e MONGO_AUTH_PASSWORD=${{ env.MONGO_AUTH_PASSWORD }} \ + -e PYTHONUNBUFFERED=TRUE \ + -e ENV=ci -d binbot_cronjobs + - name: Tag image + if: ${{ github.actor != 'dependabot[bot]' }} + run: | + docker commit binbot_cronjobs carloswufei/binbot_cronjobs & + docker tag binbot_cronjobs carloswufei/binbot_cronjobs + - name: Push to Docker Hub + if: ${{ github.actor != 'dependabot[bot]' }} + run: | + docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} + docker push carloswufei/binbot_cronjobs + python_tests: name: Python code tests diff --git a/.vscode/launch.json b/.vscode/launch.json index 83cfd2aa2..74c69efee 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,6 +20,14 @@ "console": "internalConsole", "justMyCode": false }, + { + "name": "Python: cronjobs", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/api/cronjobs.py", + "console": "internalConsole", + "justMyCode": true + }, { "type": "chrome", "request": "launch", @@ -56,14 +64,6 @@ "program": "binquant/consumer/__init__.py", "console": "internalConsole", "justMyCode": true - }, - { - "name": "Python: Test Producer", - "type": "python", - "request": "launch", - "program": "binquant/kafka_producer.py", - "console": "internalConsole", - "justMyCode": true - }, + } ] } diff --git a/Dockerfile.cronjobs b/Dockerfile.cronjobs new file mode 100644 index 000000000..f17f93f78 --- /dev/null +++ b/Dockerfile.cronjobs @@ -0,0 +1,9 @@ +FROM ubuntu:latest +RUN apt-get update && apt-get install -y --no-install-recommends python3-pip build-essential python3-dev python-setuptools +COPY api api +WORKDIR api +RUN pip3 install pipenv --no-cache-dir --upgrade +RUN pipenv install --system --deploy --ignore-pipfile --clear +ENTRYPOINT ["python3", "-u", "cronjobs.py"] + +STOPSIGNAL SIGTERM diff --git a/api/cronjobs.py b/api/cronjobs.py new file mode 100644 index 000000000..db9cc2c20 --- /dev/null +++ b/api/cronjobs.py @@ -0,0 +1,65 @@ +import atexit +import logging +import time + +from apscheduler.schedulers.background import BackgroundScheduler +from account.assets import Assets + + +logging.Formatter.converter = time.gmtime # date time in GMT/UTC +logging.basicConfig( + level=logging.INFO, + filename=None, + format="%(asctime)s.%(msecs)03d UTC %(levelname)s %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) + +scheduler = BackgroundScheduler() +assets = Assets() +timezone = "Europe/Madrid" + +scheduler.add_job( + func=assets.store_balance, + trigger="interval", + timezone=timezone, + # hour=5, + minutes=1, + id="store_balance", +) +# scheduler.add_job( +# func=assets.store_balance, +# trigger="cron", +# timezone=timezone, +# hour=5, +# minute=21, +# id="store_balance", +# ) +scheduler.add_job( + func=assets.disable_isolated_accounts, + trigger="cron", + timezone=timezone, + hour=2, + minute=1, + id="disable_isolated_accounts", +) +scheduler.add_job( + func=assets.clean_balance_assets, + trigger="cron", + timezone=timezone, + hour=3, + minute=1, + id="clean_balance_assets", +) + +try: + scheduler.start() + print("Scheduler started") + while True: + time.sleep(1) + print(scheduler.get_jobs()) + +except Exception as e: + print(e) + atexit.register(lambda: scheduler.shutdown(wait=False)) + + \ No newline at end of file diff --git a/api/market_updates.py b/api/market_updates.py index 8088a7bce..8eaf5bbc1 100644 --- a/api/market_updates.py +++ b/api/market_updates.py @@ -6,10 +6,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from streaming.streaming_controller import StreamingController from account.assets import Assets -from websocket import ( - WebSocketException, - WebSocketConnectionClosedException, -) +from websocket import WebSocketConnectionClosedException logging.Formatter.converter = time.gmtime # date time in GMT/UTC @@ -20,48 +17,8 @@ datefmt="%Y-%m-%d %H:%M:%S", ) -if os.getenv("ENV") != "ci": - - scheduler = BackgroundScheduler() - assets = Assets() - timezone = "Europe/Madrid" - - scheduler.add_job( - func=assets.store_balance, - trigger="interval", - timezone=timezone, - # hour=5, - minutes=10, - id="store_balance", - ) - # scheduler.add_job( - # func=assets.store_balance, - # trigger="cron", - # timezone=timezone, - # hour=5, - # minute=21, - # id="store_balance", - # ) - scheduler.add_job( - func=assets.disable_isolated_accounts, - trigger="cron", - timezone=timezone, - hour=2, - minute=1, - id="disable_isolated_accounts", - ) - scheduler.add_job( - func=assets.clean_balance_assets, - trigger="cron", - timezone=timezone, - hour=3, - minute=1, - id="clean_balance_assets", - ) - - scheduler.start() - try: + mu = StreamingController() mu.get_klines() @@ -69,12 +26,8 @@ logging.error("Lost websocket connection") mu = StreamingController() mu.get_klines() - - # atexit.register(lambda: scheduler.shutdown(wait=False)) except Exception as error: logging.error(f"Streaming controller error: {error}") mu = StreamingController() mu.get_klines() - - # atexit.register(lambda: scheduler.shutdown(wait=False)) diff --git a/docker-compose.yml b/docker-compose.yml index 0340f36f8..93ae9b7cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,5 +40,19 @@ services: # - api # - db + + crobjobs: + build: + context: . + dockerfile: Dockerfile.cronjobs + image: binbot_cronjobs + env_file: + - .env + restart: on-failure + container_name: binbot_cronjobs + depends_on: + # - api + - db + volumes: mongo_data: