Initial typescript setup

.github/workflows/pr.yml

@@ -97,6 +97,35 @@ jobs:
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push carloswufei/binbot_streaming
 
+  deploy_cronjobs:
+    name: Deploy cronjobs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Build image
+        run: docker build --tag binbot_cronjobs -f Dockerfile.cronjobs .
+      - name: Test run script
+        run: |
+          docker run --network host --name binbot_cronjobs \
+          -e MONGO_HOSTNAME=${{ env.MONGO_HOSTNAME }} \
+          -e MONGO_PORT=${{ env.MONGO_PORT }} \
+          -e MONGO_APP_DATABASE=${{ env.MONGO_APP_DATABASE }} \
+          -e MONGO_AUTH_USERNAME=${{ env.MONGO_AUTH_USERNAME }} \
+          -e MONGO_AUTH_PASSWORD=${{ env.MONGO_AUTH_PASSWORD }} \
+          -e PYTHONUNBUFFERED=TRUE \
+          -e ENV=ci -d binbot_cronjobs
+      - name: Tag image
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          docker commit binbot_cronjobs carloswufei/binbot_cronjobs &
+          docker tag binbot_cronjobs carloswufei/binbot_cronjobs
+      - name: Push to Docker Hub
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push carloswufei/binbot_cronjobs
+
 
   python_tests:
     name: Python code tests

.vscode/launch.json

@@ -18,6 +18,14 @@
       "request": "launch",
       "program": "${workspaceFolder}/api/market_updates.py",
       "console": "internalConsole",
+      "justMyCode": false
+    },
+    {
+      "name": "Python: cronjobs",
+      "type": "python",
+      "request": "launch",
+      "program": "${workspaceFolder}/api/cronjobs.py",
+      "console": "internalConsole",
       "justMyCode": true
     },
     {
@@ -56,14 +64,6 @@
       "program": "binquant/consumer/__init__.py",
       "console": "internalConsole",
       "justMyCode": true
-    },
-    {
-      "name": "Python: Test Producer",
-      "type": "python",
-      "request": "launch",
-      "program": "binquant/kafka_producer.py",
-      "console": "internalConsole",
-      "justMyCode": true
-    },
+    }
   ]
 }

Dockerfile.cronjobs

@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+RUN apt-get update && apt-get install -y --no-install-recommends python3-pip build-essential python3-dev python-setuptools
+COPY api api
+WORKDIR api
+RUN pip3 install pipenv --no-cache-dir --upgrade
+RUN pipenv install --system --deploy --ignore-pipfile --clear
+ENTRYPOINT ["python3", "-u", "cronjobs.py"]
+
+STOPSIGNAL SIGTERM

api/account/account.py

@@ -220,15 +220,6 @@ def min_notional_by_symbol(self, symbol, min_notional_limit="minNotional"):
         )
         return min_notional_filter[min_notional_limit]
 
-    def get_qty_precision(self, pair):
-        lot_size_by_symbol = self.lot_size_by_symbol(pair, "stepSize")
-        qty_precision = -(
-            Decimal(str(lot_size_by_symbol))
-            .as_tuple()
-            .exponent
-        )
-        return qty_precision
-
     def get_one_balance(self, symbol="BTC"):
         # Response after request
         data = self.bb_request(url=self.bb_balance_url)

api/account/assets.py

@@ -9,14 +9,13 @@
 from db import setup_db
 from tools.handle_error import json_response, json_response_error, json_response_message
 from tools.round_numbers import round_numbers
-from tools.exceptions import BinanceErrors, InvalidSymbol
+from tools.exceptions import BinanceErrors, InvalidSymbol, MarginLoanNotFound
 from deals.base import BaseDeal
 
 class Assets(BaseDeal):
     def __init__(self):
         self.db = setup_db()
         self.usd_balance = 0
-        self.isolated_balance = None
 
     def get_raw_balance(self, asset=None):
         """
@@ -338,7 +337,7 @@ async def get_balance_series(self, end_date, start_date):
         )
         return resp
 
-    async def clean_balance_assets(self):
+    def clean_balance_assets(self):
         """
         Check if there are many small assets (0.000.. BTC)
         if there are more than 5 (number of bots)
@@ -358,37 +357,42 @@ async def clean_balance_assets(self):
 
         return resp
 
-    async def disable_isolated_accounts(self):
+    def disable_isolated_accounts(self, symbol=None):
         """
         Check and disable isolated accounts
         """
         info = self.signed_request(url=self.isolated_account_url, payload={})
+        msg = "Disabling isolated margin account not required yet."
         for item in info["assets"]:
+            # Liquidate price = 0 guarantees there is no loan unpaid
             if float(item["liquidatePrice"]) == 0:
+                if float(item["baseAsset"]["free"]) > 0:
+                    self.transfer_isolated_margin_to_spot(asset=item["baseAsset"]["asset"], symbol=item["symbol"], amount=float(item["baseAsset"]["free"]))
+
+                if float(item["quoteAsset"]["free"]) > 0:
+                    self.transfer_isolated_margin_to_spot(asset=item["quoteAsset"]["asset"], symbol=item["symbol"], amount=float(item["quoteAsset"]["free"]))
+
                 self.disable_isolated_margin_account(item["symbol"])
                 msg = "Sucessfully finished disabling isolated margin accounts."
-        else:
-            msg = "Disabling isolated margin account not required yet."
 
-        resp = json_response_message(msg)
-        return resp
+        return json_response_message(msg)
 
     def one_click_liquidation(self, pair: str):
         """
         Emulate Binance Dashboard
         One click liquidation function
+
+        This endpoint is different than the margin_liquidation function
+        in that it contains some clean up functionality in the cases
+        where there are are still funds in the isolated pair
         """
-
-        try:
-            self.margin_liquidation(pair)
 
+        try:
+            self.symbol = pair
+            self.margin_liquidation(pair, self.qty_precision)
+            return json_response_message(f"Successfully liquidated {pair}")
+        except MarginLoanNotFound as error:
+            return json_response_message(f"{error}. Successfully cleared isolated pair {pair}")
         except BinanceErrors as error:
-            if error.code == -3041:
-                # Most likely not enough funds to pay back
-                # Get fiat (USDT) to pay back
-                return json_response_error(error.message)
-            if error.code == -3015:
-                # false alarm
-                pass
-            if error.code == -3051:
-                return json_response_error(error.message)
+            return json_response_error(f"Error liquidating {pair}: {error.message}")
+

api/account/routes.py

@@ -81,13 +81,13 @@ async def get_balance_series():
     return await Assets().get_balance_series(start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today))
 
 @account_blueprint.get("/clean", response_model=BalanceSeriesResponse, tags=["assets"])
-async def clean_balance():
-    return await Assets().clean_balance_assets()
+def clean_balance():
+    return Assets().clean_balance_assets()
 
 @account_blueprint.get("/disable-isolated", response_model=BalanceSeriesResponse, tags=["assets"])
-async def disable_isolated():
-    return await Assets().disable_isolated_accounts()
+def disable_isolated():
+    return Assets().disable_isolated_accounts()
 
-@account_blueprint.get("/one-click-liquidation/{asset}", response_model=BalanceSeriesResponse, tags=["assets"])
+@account_blueprint.get("/one-click-liquidation/{asset}", tags=["assets"])
 def one_click_liquidation(asset):
     return Assets().one_click_liquidation(asset)

api/apis.py

@@ -53,6 +53,7 @@ class BinanceApi:
     margin_repay_url = f"{BASE}/sapi/v1/margin/repay"
     isolated_hourly_interest = f"{BASE}/sapi/v1/margin/next-hourly-interest-rate"
     margin_order = f"{BASE}/sapi/v1/margin/order"
+    max_borrow_url = f"{BASE}/sapi/v1/margin/maxBorrowable"
 
     def signed_request(self, url, method="GET", payload={}, params={}):
         """
@@ -95,6 +96,12 @@ def enable_isolated_margin_account(self, symbol):
         return self.signed_request(self.isolated_account_url, method="POST", payload={"symbol": symbol})
 
     def disable_isolated_margin_account(self, symbol):
+        """
+        Very high weight, use as little as possible
+
+        There is a cronjob that disables all margin isolated accounts everyday
+        check market_updates
+        """
         return self.signed_request(self.isolated_account_url, method="DELETE", payload={"symbol": symbol})
 
     def transfer_isolated_margin_to_spot(self, asset, symbol, amount):
@@ -111,6 +118,9 @@ def create_margin_loan(self, asset, symbol, amount, isIsolated=True):
 
         return self.signed_request(self.loan_record_url, method="POST", payload={"asset": asset, "symbol": symbol, "amount": amount, "isIsolated": isIsolated})
 
+    def get_max_borrow(self, asset, isolated_symbol:str | None = None):
+        return self.signed_request(self.max_borrow_url, payload={"asset": asset, "isolatedSymbol":  isolated_symbol })
+
     def get_margin_loan_details(self, asset: str, isolatedSymbol: str):
         return self.signed_request(self.loan_record_url, payload={"asset": asset, "isolatedSymbol": isolatedSymbol})
 

api/bots/controllers.py

@@ -6,10 +6,9 @@
 from fastapi.exceptions import RequestValidationError
 
 from account.account import Account
-from deals.margin import MarginShortError
 from deals.controllers import CreateDealController
 from tools.enum_definitions import BinbotEnums
-from tools.exceptions import OpenDealError, NotEnoughFunds, QuantityTooLow, IsolateBalanceError
+from tools.exceptions import BinanceErrors, BinbotErrors, QuantityTooLow
 from tools.handle_error import (
     handle_binance_errors,
     json_response,
@@ -117,6 +116,14 @@ def create(self, data):
             bot = data.dict()
             bot["id"] = str(ObjectId())
 
+            # if bot["strategy"] == "margin_short":
+            #     asset = bot["pair"].replace(bot["balance_to_use"], "")
+            #     # price = self.matching_engine(bot["pair"], True)
+            #     total = float(bot["base_order_size"])
+            #     check_max_borrow = self.get_max_borrow(asset, isolated_symbol=bot["pair"])
+            #     if total > check_max_borrow["borrowLimit"]:
+            #         return json_response_error(f"Max margin account borrow limit reached")
+
             self.db_collection.insert_one(bot)
             resp = json_response(
                 {
@@ -184,14 +191,11 @@ def activate(self, botId: str):
                     bot, db_collection=self.db_collection.name
                 ).open_deal()
                 return json_response_message("Successfully activated bot!")
-            except OpenDealError as error:
-                return json_response_error(error.args[0])
-            except NotEnoughFunds as e:
-                return json_response_error(e.args[0])
-            except MarginShortError as error:
-                message = str("Unable to create margin_short bot: ".join(error.args))
-                return json_response_error(message)
-            except IsolateBalanceError as error:
+            except BinanceErrors as error:
+                self.post_errors_by_id(botId, error.message)
+                return json_response_error(error.message)
+            except BinbotErrors as error:
+                self.post_errors_by_id(botId, error.message)
                 return json_response_error(error.message)
             except Exception as error:
                 resp = json_response_error(f"Unable to activate bot: {error}")
@@ -355,3 +359,20 @@ def put_archive(self, botId):
             resp = json_response({"message": f"Failed to archive bot {error}"})
 
         return resp
+
+    def post_errors_by_id(self, bot_id: str, reported_error: str):
+        """
+        Directly post errors to Bot
+        which should show in the BotForm page in Web
+        """
+        try:
+            self.db_collection.update_one(
+                {"id": bot_id}, {"$push": {"errors": reported_error}}
+            )
+            return json_response(
+                {"message": "Successfully submitted bot errors", "botId": bot_id}
+            )
+        except Exception as error:
+            resp = json_response({"message": f"Failed to submit bot errors: {error}"})
+
+        return resp

api/bots/routes.py

@@ -63,3 +63,8 @@ def deactivate(id: str):
 @bot_blueprint.put("/bot/archive/{id}", tags=["bots"])
 def archive(id: str):
     return Bot(collection_name="bots").put_archive(id)
+
+
+@bot_blueprint.post("/bot/errors/{bot_id}", tags=["bots"])
+def bot_errors(bot_id: str, bot_errors: str):
+    return Bot(collection_name="bots").post_errors_by_id(bot_id, bot_errors)

api/bots/schemas.py

@@ -1,5 +1,5 @@
 from time import time
-from typing import Literal
+from typing import List, Literal
 
 from bson.objectid import ObjectId
 from tools.enum_definitions import Status
@@ -40,7 +40,7 @@ class BotSchema(BaseModel):
     created_at: float = time() * 1000
     deal: DealSchema = Field(default_factory=DealSchema)
     dynamic_trailling: bool = False
-    errors: list[str] = []
+    errors: list[str] = [] # Event logs
     locked_so_funds: float = 0  # funds locked by Safety orders
     mode: str = "manual"  # Manual is triggered by the terminal dashboard, autotrade by research app
     name: str = "Default bot"
@@ -90,12 +90,18 @@ def check_trailling(cls, v: str | bool):
             return False
         return True
 
+    @validator("errors")
+    def check_errors_format(cls, v: list[str]):
+        if isinstance(v, list):
+            return []
+        return []
+
     class Config:
         use_enum_values = True
         arbitrary_types_allowed = True
         json_encoders = {ObjectId: str}
         schema_extra = {
-            "description": "Most fields are optional. Deal field is generated internally, orders are filled up by Binance and",
+            "description": "Most fields are optional. Deal field is generated internally, orders are filled up by Binance",
             "example": {
                 "pair": "BNBUSDT",
                 "balance_size_to_use": 0,