Refactor cronjobs as a separate service

.github/workflows/pr.yml

@@ -97,6 +97,35 @@ jobs:
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push carloswufei/binbot_streaming
 
+  deploy_cronjobs:
+    name: Deploy cronjobs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Build image
+        run: docker build --tag binbot_cronjobs -f Dockerfile.cronjobs .
+      - name: Test run script
+        run: |
+          docker run --network host --name binbot_cronjobs \
+          -e MONGO_HOSTNAME=${{ env.MONGO_HOSTNAME }} \
+          -e MONGO_PORT=${{ env.MONGO_PORT }} \
+          -e MONGO_APP_DATABASE=${{ env.MONGO_APP_DATABASE }} \
+          -e MONGO_AUTH_USERNAME=${{ env.MONGO_AUTH_USERNAME }} \
+          -e MONGO_AUTH_PASSWORD=${{ env.MONGO_AUTH_PASSWORD }} \
+          -e PYTHONUNBUFFERED=TRUE \
+          -e ENV=ci -d binbot_cronjobs
+      - name: Tag image
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          docker commit binbot_cronjobs carloswufei/binbot_cronjobs &
+          docker tag binbot_cronjobs carloswufei/binbot_cronjobs
+      - name: Push to Docker Hub
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push carloswufei/binbot_cronjobs
+
 
   python_tests:
     name: Python code tests

.vscode/launch.json

@@ -20,6 +20,14 @@
       "console": "internalConsole",
       "justMyCode": false
     },
+    {
+      "name": "Python: cronjobs",
+      "type": "python",
+      "request": "launch",
+      "program": "${workspaceFolder}/api/cronjobs.py",
+      "console": "internalConsole",
+      "justMyCode": true
+    },
     {
       "type": "chrome",
       "request": "launch",
@@ -56,14 +64,6 @@
       "program": "binquant/consumer/__init__.py",
       "console": "internalConsole",
       "justMyCode": true
-    },
-    {
-      "name": "Python: Test Producer",
-      "type": "python",
-      "request": "launch",
-      "program": "binquant/kafka_producer.py",
-      "console": "internalConsole",
-      "justMyCode": true
-    },
+    }
   ]
 }

Dockerfile.cronjobs

@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+RUN apt-get update && apt-get install -y --no-install-recommends python3-pip build-essential python3-dev python-setuptools
+COPY api api
+WORKDIR api
+RUN pip3 install pipenv --no-cache-dir --upgrade
+RUN pipenv install --system --deploy --ignore-pipfile --clear
+ENTRYPOINT ["python3", "-u", "cronjobs.py"]
+
+STOPSIGNAL SIGTERM

api/cronjobs.py

@@ -0,0 +1,65 @@
+import atexit
+import logging
+import time
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from account.assets import Assets
+
+
+logging.Formatter.converter = time.gmtime  # date time in GMT/UTC
+logging.basicConfig(
+    level=logging.INFO,
+    filename=None,
+    format="%(asctime)s.%(msecs)03d UTC %(levelname)s %(name)s: %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+scheduler = BackgroundScheduler()
+assets = Assets()
+timezone = "Europe/Madrid"
+
+scheduler.add_job(
+    func=assets.store_balance,
+    trigger="interval",
+    timezone=timezone,
+    # hour=5,
+    minutes=1,
+    id="store_balance",
+)
+# scheduler.add_job(
+#     func=assets.store_balance,
+#     trigger="cron",
+#     timezone=timezone,
+#     hour=5,
+#     minute=21,
+#     id="store_balance",
+# )
+scheduler.add_job(
+    func=assets.disable_isolated_accounts,
+    trigger="cron",
+    timezone=timezone,
+    hour=2,
+    minute=1,
+    id="disable_isolated_accounts",
+)
+scheduler.add_job(
+    func=assets.clean_balance_assets,
+    trigger="cron",
+    timezone=timezone,
+    hour=3,
+    minute=1,
+    id="clean_balance_assets",
+)
+
+try:
+    scheduler.start()
+    print("Scheduler started")
+    while True:
+        time.sleep(1)
+        print(scheduler.get_jobs())
+
+except Exception as e:
+    print(e)
+    atexit.register(lambda: scheduler.shutdown(wait=False))
+
+

api/market_updates.py

@@ -6,10 +6,7 @@
 from apscheduler.schedulers.background import BackgroundScheduler
 from streaming.streaming_controller import StreamingController
 from account.assets import Assets
-from websocket import (
-    WebSocketException,
-    WebSocketConnectionClosedException,
-)
+from websocket import WebSocketConnectionClosedException
 
 
 logging.Formatter.converter = time.gmtime  # date time in GMT/UTC
@@ -20,61 +17,17 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-if os.getenv("ENV") != "ci":
-
-    scheduler = BackgroundScheduler()
-    assets = Assets()
-    timezone = "Europe/Madrid"
-
-    scheduler.add_job(
-        func=assets.store_balance,
-        trigger="interval",
-        timezone=timezone,
-        # hour=5,
-        minutes=10,
-        id="store_balance",
-    )
-    # scheduler.add_job(
-    #     func=assets.store_balance,
-    #     trigger="cron",
-    #     timezone=timezone,
-    #     hour=5,
-    #     minute=21,
-    #     id="store_balance",
-    # )
-    scheduler.add_job(
-        func=assets.disable_isolated_accounts,
-        trigger="cron",
-        timezone=timezone,
-        hour=2,
-        minute=1,
-        id="disable_isolated_accounts",
-    )
-    scheduler.add_job(
-        func=assets.clean_balance_assets,
-        trigger="cron",
-        timezone=timezone,
-        hour=3,
-        minute=1,
-        id="clean_balance_assets",
-    )
-
-    scheduler.start()
-
 try:
+
     mu = StreamingController()
     mu.get_klines()
 
 except WebSocketConnectionClosedException as e:
     logging.error("Lost websocket connection")
     mu = StreamingController()
     mu.get_klines()
-
-    # atexit.register(lambda: scheduler.shutdown(wait=False))
 
 except Exception as error:
     logging.error(f"Streaming controller error: {error}")
     mu = StreamingController()
     mu.get_klines()
-
-    # atexit.register(lambda: scheduler.shutdown(wait=False))

docker-compose.yml

@@ -40,5 +40,19 @@ services:
   #     - api
   #     - db
 
+
+  crobjobs:
+    build:
+      context: .
+      dockerfile: Dockerfile.cronjobs
+    image: binbot_cronjobs
+    env_file:
+      - .env
+    restart: on-failure
+    container_name: binbot_cronjobs
+    depends_on:
+      # - api
+      - db
+
 volumes:
   mongo_data: