Refactor cronjobs as a separate service

.github/workflows/pr.yml

@@ -97,6 +97,35 @@ jobs:
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push carloswufei/binbot_streaming
 
+  deploy_cronjobs:
+    name: Deploy cronjobs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Build image
+        run: docker build --tag binbot_cronjobs -f Dockerfile.cronjobs .
+      - name: Test run script
+        run: |
+          docker run --network host --name binbot_cronjobs \
+          -e MONGO_HOSTNAME=${{ env.MONGO_HOSTNAME }} \
+          -e MONGO_PORT=${{ env.MONGO_PORT }} \
+          -e MONGO_APP_DATABASE=${{ env.MONGO_APP_DATABASE }} \
+          -e MONGO_AUTH_USERNAME=${{ env.MONGO_AUTH_USERNAME }} \
+          -e MONGO_AUTH_PASSWORD=${{ env.MONGO_AUTH_PASSWORD }} \
+          -e PYTHONUNBUFFERED=TRUE \
+          -e ENV=ci -d binbot_cronjobs
+      - name: Tag image
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          docker commit binbot_cronjobs carloswufei/binbot_cronjobs &
+          docker tag binbot_cronjobs carloswufei/binbot_cronjobs
+      - name: Push to Docker Hub
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push carloswufei/binbot_cronjobs
+
 
   python_tests:
     name: Python code tests

.vscode/launch.json

@@ -20,6 +20,14 @@
       "console": "internalConsole",
       "justMyCode": false
     },
+    {
+      "name": "Python: cronjobs",
+      "type": "python",
+      "request": "launch",
+      "program": "${workspaceFolder}/api/cronjobs.py",
+      "console": "internalConsole",
+      "justMyCode": true
+    },
     {
       "type": "chrome",
       "request": "launch",
@@ -56,14 +64,6 @@
       "program": "binquant/consumer/__init__.py",
       "console": "internalConsole",
       "justMyCode": true
-    },
-    {
-      "name": "Python: Test Producer",
-      "type": "python",
-      "request": "launch",
-      "program": "binquant/kafka_producer.py",
-      "console": "internalConsole",
-      "justMyCode": true
-    },
+    }
   ]
 }

Dockerfile.cronjobs

@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+RUN apt-get update && apt-get install -y --no-install-recommends python3-pip build-essential python3-dev python-setuptools
+COPY api api
+WORKDIR api
+RUN pip3 install pipenv --no-cache-dir --upgrade
+RUN pipenv install --system --deploy --ignore-pipfile --clear
+ENTRYPOINT ["python3", "-u", "cronjobs.py"]
+
+STOPSIGNAL SIGTERM

api/account/assets.py

@@ -337,7 +337,7 @@ async def get_balance_series(self, end_date, start_date):
         )
         return resp
 
-    async def clean_balance_assets(self):
+    def clean_balance_assets(self):
         """
         Check if there are many small assets (0.000.. BTC)
         if there are more than 5 (number of bots)
@@ -357,7 +357,7 @@ async def clean_balance_assets(self):
 
         return resp
 
-    async def disable_isolated_accounts(self, symbol=None):
+    def disable_isolated_accounts(self, symbol=None):
         """
         Check and disable isolated accounts
         """

api/account/routes.py

@@ -81,12 +81,12 @@ async def get_balance_series():
     return await Assets().get_balance_series(start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today))
 
 @account_blueprint.get("/clean", response_model=BalanceSeriesResponse, tags=["assets"])
-async def clean_balance():
-    return await Assets().clean_balance_assets()
+def clean_balance():
+    return Assets().clean_balance_assets()
 
 @account_blueprint.get("/disable-isolated", response_model=BalanceSeriesResponse, tags=["assets"])
-async def disable_isolated():
-    return await Assets().disable_isolated_accounts()
+def disable_isolated():
+    return Assets().disable_isolated_accounts()
 
 @account_blueprint.get("/one-click-liquidation/{asset}", tags=["assets"])
 def one_click_liquidation(asset):

api/cronjobs.py

@@ -0,0 +1,52 @@
+import atexit
+import logging
+import time
+import asyncio
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+from account.assets import Assets
+
+
+logging.Formatter.converter = time.gmtime  # date time in GMT/UTC
+logging.basicConfig(
+    level=logging.INFO,
+    filename=None,
+    format="%(asctime)s.%(msecs)03d UTC %(levelname)s %(name)s: %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+def main():
+
+    scheduler = BlockingScheduler()
+    assets = Assets()
+    timezone = "Europe/London"
+
+    scheduler.add_job(
+        func=assets.store_balance,
+        trigger="cron",
+        timezone=timezone,
+        hour=1,
+        minute=1,
+        id="store_balance",
+    )
+    scheduler.add_job(
+        func=assets.disable_isolated_accounts,
+        trigger="cron",
+        timezone=timezone,
+        hour=2,
+        minute=1,
+        id="disable_isolated_accounts",
+    )
+    scheduler.add_job(
+        func=assets.clean_balance_assets,
+        trigger="cron",
+        timezone=timezone,
+        hour=3,
+        minute=1,
+        id="clean_balance_assets",
+    )
+    scheduler.start()
+
+
+if __name__ == "__main__":
+    main()

api/deals/margin.py

@@ -370,10 +370,15 @@ def margin_short_base_order(self):
 
         if self.db_collection.name == "bots":
             self.init_margin_short(initial_price)
-            order_res = self.sell_margin_order(
-                symbol=self.active_bot.pair,
-                qty=self.active_bot.deal.margin_short_base_order,
-            )
+            try:
+                order_res = self.sell_margin_order(
+                    symbol=self.active_bot.pair,
+                    qty=self.active_bot.deal.margin_short_base_order,
+                )
+            except BinanceErrors as error:
+                if error.code == -3052:
+                    print(error)
+                    return
         else:
             # Simulate Margin sell
             # qty doesn't matter in paper bots

api/market_updates.py

@@ -6,10 +6,7 @@
 from apscheduler.schedulers.background import BackgroundScheduler
 from streaming.streaming_controller import StreamingController
 from account.assets import Assets
-from websocket import (
-    WebSocketException,
-    WebSocketConnectionClosedException,
-)
+from websocket import WebSocketConnectionClosedException
 
 
 logging.Formatter.converter = time.gmtime  # date time in GMT/UTC
@@ -20,61 +17,17 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-if os.getenv("ENV") != "ci":
-
-    scheduler = BackgroundScheduler()
-    assets = Assets()
-    timezone = "Europe/Madrid"
-
-    scheduler.add_job(
-        func=assets.store_balance,
-        trigger="interval",
-        timezone=timezone,
-        # hour=5,
-        minutes=10,
-        id="store_balance",
-    )
-    # scheduler.add_job(
-    #     func=assets.store_balance,
-    #     trigger="cron",
-    #     timezone=timezone,
-    #     hour=5,
-    #     minute=21,
-    #     id="store_balance",
-    # )
-    scheduler.add_job(
-        func=assets.disable_isolated_accounts,
-        trigger="cron",
-        timezone=timezone,
-        hour=2,
-        minute=1,
-        id="disable_isolated_accounts",
-    )
-    scheduler.add_job(
-        func=assets.clean_balance_assets,
-        trigger="cron",
-        timezone=timezone,
-        hour=3,
-        minute=1,
-        id="clean_balance_assets",
-    )
-
-    scheduler.start()
-
 try:
+
     mu = StreamingController()
     mu.get_klines()
 
 except WebSocketConnectionClosedException as e:
     logging.error("Lost websocket connection")
     mu = StreamingController()
     mu.get_klines()
-
-    # atexit.register(lambda: scheduler.shutdown(wait=False))
 
 except Exception as error:
     logging.error(f"Streaming controller error: {error}")
     mu = StreamingController()
     mu.get_klines()
-
-    # atexit.register(lambda: scheduler.shutdown(wait=False))

docker-compose.yml

@@ -40,5 +40,19 @@ services:
   #     - api
   #     - db
 
+
+  # crobjobs:
+  #   build:
+  #     context: .
+  #     dockerfile: Dockerfile.cronjobs
+  #   image: binbot_cronjobs
+  #   env_file:
+  #     - .env
+  #   restart: on-failure
+  #   container_name: binbot_cronjobs
+  #   depends_on:
+  #     # - api
+  #     - db
+
 volumes:
   mongo_data: