Stable maintenance (#495)

* Trigger bot save and activation sequentially in the update deal action

* Update streaming logging

* Refactor create deal streaming into a separate spot streaming updates class

* Add long bot autoswitch to margin short bot

* Fix trade price division

* Fix transfer qty to isolated in order to cover stop loss

* Retry loan repayment logic

* Fix bots not updating after deal is opened (update deal)

* Fix login

* Use current price for switch to long bot, so that out of date bots don't show bloated profits

* Fix long bots streaming updates

* Remove short buy from UI

* Add isolated balance to db.balance

* Add benchmark graph data endpoint

* Build benchmark component and redux data

* Profit and Loss data and visualization

* Adjustments for real time balance calculations

* Improve errors on failure

* Clean up Web application

---------

Co-authored-by: Carlos Wu Fei <[email protected]>

.gitignore

@@ -7,6 +7,7 @@
 .vscode/settings.json
 .venv/
 binbot-research/
+binquant/
 
 ##
 #

.vscode/launch.json

@@ -42,16 +42,28 @@
       "console": "internalConsole"
     },
     {
-      "type": "pwa-chrome",
+      "name": "Python: Producer",
+      "type": "python",
       "request": "launch",
-      "name": "React: Web",
-      "url": "http://localhost:3000",
-      "webRoot": "${workspaceFolder}/web/src",
-      "preLaunchTask": "startApp",
-      "postDebugTask": "stopApp",
-      "sourceMapPathOverrides": {
-        "webpack:///src/*": "${webRoot}/*"
-      }
-    }
+      "program": "binquant/producer.py",
+      "console": "internalConsole",
+      "justMyCode": true
+    },
+    {
+      "name": "Python: Consumer",
+      "type": "python",
+      "request": "launch",
+      "program": "binquant/consumer/__init__.py",
+      "console": "internalConsole",
+      "justMyCode": true
+    },
+    {
+      "name": "Python: Test Producer",
+      "type": "python",
+      "request": "launch",
+      "program": "binquant/kafka_producer.py",
+      "console": "internalConsole",
+      "justMyCode": true
+    },
   ]
 }

api/account/account.py

@@ -12,6 +12,7 @@
 from pymongo import MongoClient
 import os
 import pandas
+
 class Account(BinbotApi):
     def __init__(self):
         self.db = setup_db()

api/account/assets.py

@@ -5,6 +5,8 @@
 from account.account import Account
 from account.schemas import BalanceSchema
 from bson.objectid import ObjectId
+from charts.models import CandlestickParams
+from charts.models import Candlestick
 from db import setup_db
 from tools.handle_error import InvalidSymbol, json_response, json_response_message
 from tools.round_numbers import round_numbers
@@ -74,7 +76,7 @@ def store_balance(self) -> dict:
         for b in bin_balance["data"]:
             # Only tether coins for hedging
             if b["asset"] == "NFT":
-                break
+                continue
             elif b["asset"] in ["USD", "USDT"]:
                 qty = self._check_locked(b)
                 total_usdt += qty
@@ -88,6 +90,10 @@ def store_balance(self) -> dict:
                     # Some coins like NFT are air dropped and cannot be traded
                     break
 
+        isolated_balance_total = self.get_isolated_balance_total()
+        rate = self.get_ticker_price("BTCUSDT")
+        total_usdt += float(isolated_balance_total) * float(rate)
+
         total_balance = {
             "time": current_time.strftime("%Y-%m-%d"),
             "balances": bin_balance["data"],
@@ -115,8 +121,10 @@ def balance_estimate(self, fiat="USDT"):
         balances_response = self.get_raw_balance()
         # Isolated m
         isolated_margin = self.signed_request(url=self.isolated_account_url)
-        get_usdt_btc_rate = self.ticker(symbol=f'BTC{fiat}', json=False)
-        total_isolated_margin = float(isolated_margin["totalNetAssetOfBtc"]) * float(get_usdt_btc_rate["price"])
+        get_usdt_btc_rate = self.ticker(symbol=f"BTC{fiat}", json=False)
+        total_isolated_margin = float(isolated_margin["totalNetAssetOfBtc"]) * float(
+            get_usdt_btc_rate["price"]
+        )
 
         balances = json.loads(balances_response.body)
         total_fiat = 0
@@ -144,7 +152,7 @@ def balance_estimate(self, fiat="USDT"):
             "total_fiat": total_fiat + total_isolated_margin,
             "total_isolated_margin": total_isolated_margin,
             "fiat_left": left_to_allocate,
-            "asset": fiat
+            "asset": fiat,
         }
         if balance:
             resp = json_response({"data": balance})
@@ -164,7 +172,6 @@ def balance_series(self, fiat="GBP", start_time=None, end_time=None, limit=5):
         )
         balances = []
         for datapoint in snapshot_account_data["snapshotVos"]:
-
             fiat_rate = self.get_ticker_price(f"BTC{fiat}")
             total_fiat = float(datapoint["data"]["totalAssetOfBtc"]) * float(fiat_rate)
             balance = {
@@ -238,3 +245,95 @@ async def retrieve_gainers_losers(self, market_asset="USDT"):
                 "data": gainers_losers_list,
             }
         )
+
+    def match_series_dates(
+        self, dates, balance_date, i: int = 0, count=0
+    ) -> int | None:
+        if i == len(dates):
+            return None
+
+
+        for idx, d in enumerate(dates):
+            dt_obj = datetime.fromtimestamp(d / 1000)
+            str_date = datetime.strftime(dt_obj, "%Y-%m-%d")
+
+            # Match balance store dates with btc price dates
+            if str_date == balance_date:
+                return idx
+        else:
+            print("Not able to match any BTC dates for this balance store date")
+            return None
+
+    async def get_balance_series(self, end_date, start_date):
+        params = {}
+
+        if start_date:
+            start_date = start_date * 1000
+            try:
+                float(start_date)
+            except ValueError:
+                resp = json_response(
+                    {"message": f"start_date must be a timestamp float", "data": []}
+                )
+                return resp
+
+            obj_start_date = datetime.fromtimestamp(int(float(start_date) / 1000))
+            gte_tp_id = ObjectId.from_datetime(obj_start_date)
+            try:
+                params["_id"]["$gte"] = gte_tp_id
+            except KeyError:
+                params["_id"] = {"$gte": gte_tp_id}
+
+        if end_date:
+            end_date = end_date * 1000
+            try:
+                float(end_date)
+            except ValueError as e:
+                resp = json_response(
+                    {"message": f"end_date must be a timestamp float: {e}", "data": []}
+                )
+                return resp
+
+            obj_end_date = datetime.fromtimestamp(int(float(end_date) / 1000))
+            lte_tp_id = ObjectId.from_datetime(obj_end_date)
+            params["_id"]["$lte"] = lte_tp_id
+
+        balance_series = list(self.db.balances.find(params).sort([("_id", -1)]))
+
+        # btc candlestick data series
+        params = CandlestickParams(
+            limit=31, # One month - 1 (calculating percentages) worth of data to display
+            symbol="BTCUSDT",
+            interval="1d",
+        )
+
+        cs = Candlestick()
+        df, dates = cs.get_klines(params)
+        trace = cs.candlestick_trace(df, dates)
+
+        balances_series_diff = []
+        balances_series_dates = []
+        balance_btc_diff = []
+        balance_series.sort(key=lambda item: item["_id"], reverse=False)
+
+        for index, item in enumerate(balance_series):
+            btc_index = self.match_series_dates(dates, item["time"], index)
+            if btc_index:
+                balances_series_diff.append(float(balance_series[index]["estimated_total_usdt"]))
+                balances_series_dates.append(item["time"])
+                balance_btc_diff.append(float(trace["close"][btc_index]))
+            else:
+                continue
+
+        resp = json_response(
+            {
+                "message": "Sucessfully rendered benchmark data.",
+                "data": {
+                    "usdt": balances_series_diff,
+                    "btc": balance_btc_diff,
+                    "dates": balances_series_dates,
+                },
+                "error": 0,
+            }
+        )
+        return resp

api/account/routes.py

@@ -1,8 +1,8 @@
 from fastapi import APIRouter
-
+from datetime import datetime, timedelta
 from account.account import Account
 from account.assets import Assets
-from account.schemas import BalanceResponse, GainersLosersResponse
+from account.schemas import BalanceResponse, GainersLosersResponse, BalanceSeriesResponse
 
 account_blueprint = APIRouter()
 
@@ -50,26 +50,32 @@ def ticker_24(pair=None):
     return Account().ticker_24(symbol=pair)
 
 
-@account_blueprint.get("/balance/estimate", tags=["account"])
+@account_blueprint.get("/balance/estimate", tags=["assets"])
 async def balance_estimated():
     return Assets().balance_estimate()
 
 
-@account_blueprint.get("/balance/series", tags=["account"])
+@account_blueprint.get("/balance/series", tags=["assets"])
 def balance_series():
     return Assets().balance_series()
 
 
-@account_blueprint.get("/pnl", tags=["account"])
+@account_blueprint.get("/pnl", tags=["assets"])
 def get_pnl():
     return Assets().get_pnl()
 
 
-@account_blueprint.get("/store-balance", tags=["account"])
+@account_blueprint.get("/store-balance", tags=["assets"])
 def store_balance():
     return Assets().store_balance()
 
 
-@account_blueprint.get("/gainers-losers", response_model=GainersLosersResponse, tags=["account"])
+@account_blueprint.get("/gainers-losers", response_model=GainersLosersResponse, tags=["assets"])
 async def retrieve_gainers_losers():
     return await Assets().retrieve_gainers_losers()
+
+@account_blueprint.get("/balance-series", response_model=BalanceSeriesResponse, tags=["assets"])
+async def get_balance_series():
+    today = datetime.now()
+    month_ago = today - timedelta(30)
+    return await Assets().get_balance_series(start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today))

api/account/schemas.py

@@ -53,6 +53,12 @@ class Binance24Ticker(BaseModel):
     count: int
 
 
+class BinanceBalance(BaseModel):
+    asset: str
+    free: float
+    locked: float
+
+
 class GainersLosersResponse(StandardResponse):
     data: list[Binance24Ticker]
 
@@ -65,3 +71,13 @@ class EstimatedBalance(BaseModel):
 
 class EstimatedBalancesResponse(StandardResponse):
     data: EstimatedBalance
+
+
+class BalanceSeries(StandardResponse):
+    usdt: list[float]
+    btc: list[float]
+    dates: list[str]
+
+
+class BalanceSeriesResponse(StandardResponse):
+    data: list[BalanceSeries]

api/apis.py

@@ -4,10 +4,9 @@
 from urllib.parse import urlencode
 from time import time
 from requests import request
-from tools.handle_error import handle_binance_errors, json_response, json_response_error
+from tools.handle_error import handle_binance_errors, json_response, json_response_error, IsolateBalanceError
 from py3cw.request import Py3CW
 
-
 class BinanceApi:
     """
     Binance API URLs
@@ -128,8 +127,26 @@ def get_isolated_balance(self, symbol=None):
         Use isolated margin account is preferrable,
         because this is the one that supports the most assets
         """
-        info = self.signed_request(url=self.isolated_account_url, payload={"symbols": symbol})
+        payload = {}
+        if symbol:
+            payload["symbols"] = [symbol]
+        info = self.signed_request(url=self.isolated_account_url, payload=payload)
         assets = info["assets"]
+        if len(assets) == 0:
+            raise IsolateBalanceError("Hit symbol 24hr restriction or not available (requires transfer in)")
+        return assets
+
+    def get_isolated_balance_total(self):
+        """
+        Get balance of Isolated Margin account
+
+        Use isolated margin account is preferrable,
+        because this is the one that supports the most assets
+        """
+        info = self.signed_request(url=self.isolated_account_url, payload={})
+        assets = info['totalNetAssetOfBtc']
+        if len(assets) == 0:
+            raise IsolateBalanceError("Hit symbol 24hr restriction or not available (requires transfer in)")
         return assets
 
 class BinbotApi(BinanceApi):

api/autotrade/controller.py

@@ -19,11 +19,12 @@ def __init__(
         self, document_id: Literal["test_autotrade_settings", "settings"] = "settings"
     ):
         self.document_id = document_id
-        self.db = setup_db().research_controller
+        self.db = setup_db()
+        self.db_collection = self.db.research_controller
 
     def get_settings(self):
         try:
-            settings = self.db.find_one({"_id": self.document_id})
+            settings = self.db_collection.find_one({"_id": self.document_id})
             resp = json_response(
                 {"message": "Successfully retrieved settings", "data": settings}
             )
@@ -40,7 +41,7 @@ def edit_settings(self, data):
             if "update_required" in settings:
                 settings["update_required"] = time()
 
-            self.db.update_one({"_id": self.document_id}, {"$set": settings})
+            self.db_collection.update_one({"_id": self.document_id}, {"$set": settings})
             resp = json_response_message("Successfully updated settings")
         except TypeError as e:
 

api/bots/controllers.py

@@ -7,13 +7,13 @@
 
 from account.account import Account
 from deals.margin import MarginShortError
-from db import setup_db
 from deals.controllers import CreateDealController
 from tools.enum_definitions import BinbotEnums
 from tools.exceptions import OpenDealError
 from tools.handle_error import (
     NotEnoughFunds,
     QuantityTooLow,
+    IsolateBalanceError,
     handle_binance_errors,
     json_response,
     json_response_message,
@@ -26,7 +26,7 @@
 
 class Bot(Account):
     def __init__(self, collection_name="paper_trading"):
-        self.db = setup_db()
+        super().__init__()
         self.db_collection = self.db[collection_name]
 
     def _update_required(self):
@@ -154,11 +154,12 @@ def edit(self, botId, data: BotSchema):
             resp = json_response(
                 {"message": "Successfully updated bot", "botId": str(botId)}
             )
-            self._update_required()
         except RequestValidationError as e:
             resp = json_response_error(f"Failed validation: {e}")
         except Exception as e:
             resp = json_response_error(f"Failed to create new bot: {e}")
+
+        self._update_required()
         return resp
 
     def delete(self, bot_ids: List[str] = Query(...)):
@@ -193,6 +194,8 @@ def activate(self, botId: str):
             except MarginShortError as error:
                 message = str("Unable to create margin_short bot: ".join(error.args))
                 return json_response_error(message)
+            except IsolateBalanceError as error:
+                return json_response_error(error.message)
             except Exception as error:
                 resp = json_response_error(f"Unable to activate bot: {error}")
                 return resp