From ac6477ac6e6cda20dc94594191d06324a44b6b52 Mon Sep 17 00:00:00 2001 From: Carlos Wu Fei Date: Sun, 8 Dec 2024 22:20:38 +0000 Subject: [PATCH] Refactor and unify simulated orders (paper trading) --- api/account/account.py | 7 +- api/bots/controllers.py | 338 --------------------- api/bots/routes.py | 96 +++--- api/database/api_db.py | 2 +- api/database/bot_crud.py | 49 ++- api/database/models/bot_table.py | 96 +++++- api/database/models/paper_trading_table.py | 94 +++++- api/database/paper_trading_crud.py | 114 ++++++- api/deals/base.py | 127 +------- api/deals/controllers.py | 181 ++++++----- api/deals/margin.py | 181 +++++------ api/deals/spot.py | 111 ++++--- api/mypy.ini | 8 +- api/orders/controller.py | 96 +++++- api/paper_trading/routes.py | 109 +++++-- api/streaming/streaming_controller.py | 173 ++++++----- api/tools/handle_error.py | 2 +- api/tools/round_numbers.py | 10 + api/user/models/user.py | 4 +- 19 files changed, 905 insertions(+), 893 deletions(-) delete mode 100644 api/bots/controllers.py diff --git a/api/account/account.py b/api/account/account.py index bc37f4ec1..80086cd63 100644 --- a/api/account/account.py +++ b/api/account/account.py @@ -1,4 +1,6 @@ import requests +import os +import pandas from apis import BinbotApi from tools.handle_error import ( handle_binance_errors, @@ -9,8 +11,6 @@ from database.db import setup_db from requests_cache import CachedSession, MongoCache from pymongo import MongoClient -import os -import pandas from decimal import Decimal @@ -253,8 +253,7 @@ def matching_engine(self, symbol: str, order_side: bool, qty=None): @param: qty - quantity wanted to be bought/sold """ - params = [("symbol", symbol)] - res = requests.get(url=self.order_book_url, params=params) + res = requests.get(url=self.order_book_url, params={"symbol": symbol}) data = handle_binance_errors(res) if order_side: diff --git a/api/bots/controllers.py b/api/bots/controllers.py deleted file mode 100644 index 1379db542..000000000 --- a/api/bots/controllers.py +++ /dev/null @@ -1,338 +0,0 @@ -from pymongo import ReturnDocument -from time import time -from datetime import datetime -from bson.objectid import ObjectId -from fastapi.exceptions import RequestValidationError -from account.account import Account -from database.db import Database -from deals.models import BinanceOrderModel, DealModel -from base_producer import BaseProducer -from tools.enum_definitions import BinbotEnums, DealType, Status, Strategy -from tools.handle_error import json_response, json_response_message, json_response_error -from typing import List -from fastapi import Query -from bots.schemas import BotSchema, ErrorsRequestBody -from deals.controllers import CreateDealController -from tools.exceptions import BinanceErrors, InsufficientBalance - - -class Bot(Database, Account): - def __init__(self, collection_name="bots"): - super().__init__() - self.db_collection = self._db[collection_name] - self.base_producer = BaseProducer() - self.producer = self.base_producer.start_producer() - self.deal: CreateDealController | None = None - - def get_active_pairs(self, symbol: str | None = None): - """ - Get distinct (non-repeating) bots by status active - """ - params = {"status": Status.active.value} - if symbol: - params["pair"] = symbol - - bots = list(self.db_collection.distinct("pair", params)) - return bots - - def get(self, status, start_date=None, end_date=None, no_cooldown=False): - """ - Get all bots in the db except archived - Args: - - archive=false - - filter_by: string - last-week, last-month, all - """ - params = {} - - if status and status in BinbotEnums.statuses: - params["status"] = status - - if start_date: - try: - float(start_date) - except ValueError: - resp = json_response( - {"message": "start_date must be a timestamp float", "data": []} - ) - return resp - - obj_start_date = datetime.fromtimestamp(int(float(start_date) / 1000)) - gte_tp_id = ObjectId.from_datetime(obj_start_date) - try: - params["_id"]["$gte"] = gte_tp_id - except KeyError: - params["_id"] = {"$gte": gte_tp_id} - - if end_date: - try: - float(end_date) - except ValueError as e: - resp = json_response( - {"message": f"end_date must be a timestamp float: {e}", "data": []} - ) - return resp - - obj_end_date = datetime.fromtimestamp(int(float(end_date) / 1000)) - lte_tp_id = ObjectId.from_datetime(obj_end_date) - params["_id"]["$lte"] = lte_tp_id - - # Only retrieve active and cooldown bots - # These bots will be removed from signals - if status and no_cooldown: - params = { - "$or": [ - {"status": status}, - { - "$where": """function () { - if (this.deal !== undefined) { - return new Date().getTime() - this.deal.sell_timestamp < (this.cooldown * 1000) - } else { - return (new Date().getTime() - this.created_at < (this.cooldown * 1000)) - } - }""" - }, - ] - } - - try: - bot = list( - self.db_collection.find(params).sort( - [("_id", -1), ("status", 1), ("pair", 1)] - ) - ) - resp = json_response({"message": "Sucessfully found bots!", "data": bot}) - except Exception as error: - resp = json_response_message(error) - - return resp - - def get_one(self, bot_id=None, symbol=None, status: Status = None): - if bot_id: - params = {"id": bot_id} - elif symbol: - params = {"pair": symbol} - else: - raise ValueError("id or symbol is required to find bot") - - if status: - params["status"] = status - - bot = self.db_collection.find_one(params) - return bot - - def create(self, data: BotSchema): - """ - Always creates new document - """ - try: - bot = data.model_dump() - bot["id"] = str(ObjectId()) - - self.db_collection.insert_one(bot) - resp = json_response( - { - "message": "Successfully created bot!", - "botId": str(bot["id"]), - } - ) - self.base_producer.update_required(self.producer, "CREATE_BOT") - - except RequestValidationError as error: - resp = json_response_error(f"Failed to create new bot: {error}") - pass - - return resp - - def edit(self, botId, data: BotSchema): - if not botId: - return json_response_message("id is required to update bot") - - try: - # Merge new data with old data - initial_bot_data = self.db_collection.find_one({"id": botId}) - # Ensure if client-side updated current_price is not overwritten - # Client side most likely has most up to date current_price because of websockets single pair update in BotDetail - if data.deal.current_price: - initial_bot_data["deal"]["current_price"] = data.deal.current_price - data.deal = initial_bot_data["deal"] - data.orders = initial_bot_data["orders"] - data.created_at = initial_bot_data["created_at"] - data.total_commission = initial_bot_data["total_commission"] - data.updated_at = round(time() * 1000) - bot = data.model_dump() - if "id" in bot: - bot.pop("id") - self.db_collection.update_one({"id": botId}, {"$set": bot}) - resp = json_response( - {"message": "Successfully updated bot", "botId": str(botId)} - ) - except RequestValidationError as e: - resp = json_response_error(f"Failed validation: {e}") - pass - - self.base_producer.update_required(self.producer, "EDIT_BOT") - return resp - - def delete(self, bot_ids: List[str] = Query(...)): - """ - Delete by multiple ids. - For a single id, pass one id in a list - """ - - try: - self.db_collection.delete_many({"id": {"$in": [id for id in bot_ids]}}) - resp = json_response_message("Successfully deleted bot(s)") - self.base_producer.update_required(self.producer, "DELETE_BOT") - except Exception as error: - resp = json_response_error(f"Failed to delete bot(s) {error}") - - return resp - - def activate(self, bot: dict | BotSchema): - if isinstance(bot, dict): - self.active_bot = BotSchema.model_validate(bot) - else: - self.active_bot = bot - - CreateDealController(self.active_bot, db_collection="bots").open_deal() - self.base_producer.update_required(self.producer, "ACTIVATE_BOT") - return bot - - def deactivate(self, bot: BotSchema) -> dict: - """ - DO NOT USE, LEGACY CODE NEEDS TO BE REVAMPED - Close all deals, sell pair and deactivate - 1. Close all deals - 2. Sell Coins - 3. Delete bot - """ - # Close all active orders - if len(bot.orders) > 0: - for d in bot.orders: - if d.status == "NEW" or d.status == "PARTIALLY_FILLED": - order_id = d.order_id - try: - self.delete_opened_order(bot.pair, order_id) - except BinanceErrors as error: - if error.code == -2011: - self.update_deal_logs( - "Order not found. Most likely not completed", bot - ) - pass - - if not bot.deal.buy_total_qty or bot.deal.buy_total_qty == 0: - msg = "Not enough balance to close and sell" - self.update_deal_logs(msg, bot) - raise InsufficientBalance(msg) - - deal_controller = CreateDealController(bot, db_collection="bots") - - if bot.strategy == Strategy.margin_short: - order_res = deal_controller.margin_liquidation(bot.pair) - panic_close_order = BinanceOrderModel( - timestamp=order_res["transactTime"], - deal_type=DealType.panic_close, - order_id=order_res["orderId"], - pair=order_res["symbol"], - order_side=order_res["side"], - order_type=order_res["type"], - price=order_res["price"], - qty=order_res["origQty"], - time_in_force=order_res["timeInForce"], - status=order_res["status"], - ) - - bot.total_commission = self.calculate_total_commissions(order_res["fills"]) - - bot.orders.append(panic_close_order) - else: - try: - res = deal_controller.spot_liquidation(bot.pair) - except InsufficientBalance as error: - self.update_deal_logs(error.message, bot) - bot.status = Status.completed - bot = self.save_bot_streaming(bot) - return bot.model_dump() - - panic_close_order = BinanceOrderModel( - timestamp=res["transactTime"], - order_id=res["orderId"], - deal_type=DealType.panic_close, - pair=res["symbol"], - order_side=res["side"], - order_type=res["type"], - price=res["price"], - qty=res["origQty"], - time_in_force=res["timeInForce"], - status=res["status"], - ) - - for chunk in res["fills"]: - bot.total_commission += float(chunk["commission"]) - - bot.orders.append(panic_close_order) - - bot.deal = DealModel( - buy_timestamp=res["transactTime"], - buy_price=res["price"], - buy_total_qty=res["origQty"], - current_price=res["price"], - ) - - bot.status = Status.completed - bot_obj = bot.model_dump() - if "_id" in bot_obj: - bot_obj.pop("_id") - - document = self.db_collection.find_one_and_update( - {"id": bot.id}, - {"$set": bot}, - return_document=ReturnDocument.AFTER, - ) - - return document - - def put_archive(self, botId): - """ - Change status to archived - """ - bot = self.db_collection.find_one({"id": botId}) - if bot["status"] == "active": - return json_response( - {"message": "Cannot archive an active bot!", "botId": botId} - ) - - if bot["status"] == "archived": - status = "inactive" - else: - status = "archived" - - try: - self.db_collection.update_one({"id": botId}, {"$set": {"status": status}}) - resp = json_response( - {"message": "Successfully archived bot", "botId": botId} - ) - return resp - except Exception as error: - resp = json_response({"message": f"Failed to archive bot {error}"}) - - return resp - - def post_errors_by_id(self, bot_id: str, reported_error: ErrorsRequestBody): - """ - Directly post errors to Bot - which should show in the BotForm page in Web - - Similar to update_deal_errors - but without a bot instance. - """ - operation = {"$push": {"errors": reported_error}} - if isinstance(reported_error, list): - operation = {"$push": {"errors": {"$each": reported_error}}} - elif isinstance(reported_error, str): - operation = {"$push": {"errors": reported_error}} - else: - raise ValueError("reported_error must be a list") - - self.db_collection.update_one({"id": bot_id}, operation) - pass diff --git a/api/bots/routes.py b/api/bots/routes.py index b27bfd2d2..7c636a891 100644 --- a/api/bots/routes.py +++ b/api/bots/routes.py @@ -1,16 +1,14 @@ -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends from sqlmodel import Session -from api.database.bot_crud import BotTableCrud -from api.deals.controllers import CreateDealController -from bots.bot_table_controller import BotTableController +from database.bot_crud import BotTableCrud +from deals.controllers import CreateDealController from database.models.bot_table import BotTable from database.utils import get_session from tools.handle_error import json_response, json_response_error, json_response_message -from bots.controllers import Bot from bots.schemas import BotSchema, BotListResponse, ErrorsRequestBody from typing import List from tools.exceptions import BinanceErrors, BinbotErrors -from tools.enum_definitions import Status +from fastapi.encoders import jsonable_encoder bot_blueprint = APIRouter() @@ -22,23 +20,30 @@ def get( start_date: float | None = None, end_date: float | None = None, no_cooldown: bool = True, + session: Session = Depends(get_session), ): - return Bot(collection_name="bots").get(status, start_date, end_date, no_cooldown) + try: + bots = BotTableCrud(session=session).get(status, start_date, end_date, no_cooldown) + return json_response({"message": "Bots found!", "data": jsonable_encoder(bots)}) + except ValueError as error: + return json_response_error(error) @bot_blueprint.get("/bot/active-pairs", tags=["bots"]) -def get_active_pairs(): +def get_active_pairs( + session: Session = Depends(get_session), +): try: - bot = Bot(collection_name="bots").get_active_pairs() + bot = BotTableCrud(session=session).get_active_pairs() return json_response({"message": "Active pairs found!", "data": bot}) except ValueError as error: return json_response_error(error) @bot_blueprint.get("/bot/{id}", tags=["bots"]) -def get_one_by_id(id: str): +def get_one_by_id(id: str, session: Session = Depends(get_session)): try: - bot = Bot(collection_name="bots").get_one(bot_id=id, symbol=None) + bot = BotTableCrud(session=session).get_one(bot_id=id, symbol=None) if not bot: return json_response_error("Bot not found.") else: @@ -48,30 +53,43 @@ def get_one_by_id(id: str): @bot_blueprint.get("/bot/{symbol}", tags=["bots"]) -def get_one_by_symbol(symbol: str): +def get_one_by_symbol( + symbol: str, + session: Session = Depends(get_session), +): try: - bot = Bot(collection_name="bots").get_one(bot_id=None, symbol=symbol) + bot = BotTableCrud(session=session).get_one(bot_id=None, symbol=symbol) return json_response({"message": "Bot found", "data": bot}) except ValueError as error: return json_response_error(error) @bot_blueprint.post("/bot", tags=["bots"]) -def create(bot_item: BotSchema): - return Bot(collection_name="bots").create(bot_item) +def create( + bot_item: BotSchema, + session: Session = Depends(get_session), +): + return BotTableCrud(session=session).create(bot_item) @bot_blueprint.put("/bot/{id}", tags=["bots"]) -def edit(id: str, bot_item: BotSchema): - return Bot(collection_name="bots").edit(id, bot_item) +def edit( + id: str, + bot_item: BotSchema, + session: Session = Depends(get_session), +): + return BotTableCrud(session=session).edit(id, bot_item) @bot_blueprint.delete("/bot", tags=["bots"]) -def delete(id: List[str]): +def delete( + id: List[str], + session: Session = Depends(get_session), +): """ Delete bots, given a list of ids """ - return Bot(collection_name="bots").delete(id) + return BotTableCrud(session=session).delete(id) @bot_blueprint.get("/bot/activate/{id}", tags=["bots"]) @@ -85,44 +103,46 @@ async def activate_by_id(id: str, session: Session = Depends(get_session)): """ bot = BotTableCrud(session=session).get_one(bot_id=id) if not bot: - return json_response_message("Successfully activated bot!") + return json_response_error("Bot not found.") bot_instance = CreateDealController(bot, db_table=BotTable) try: - bot_instance.activate(bot) + bot_instance.open_deal() return json_response_message("Successfully activated bot!") except BinbotErrors as error: - bot_instance.update_logs(bot_id=id, log_message=error.message) + bot_instance.controller.update_logs(bot_id=id, log_message=error.message) return json_response_error(error.message) except BinanceErrors as error: - bot_instance.update_logs(bot_id=id, log_message=error.message) + bot_instance.controller.update_logs(bot_id=id, log_message=error.message) return json_response_error(error.message) @bot_blueprint.delete("/bot/deactivate/{id}", tags=["bots"]) -def deactivation(id: str): +def deactivation(id: str, session: Session = Depends(get_session)): """ Deactivation means closing all deals and selling to fiat. This is often used to prevent losses """ - botModel = Bot(collection_name="bots") - bot = botModel.db_collection.find_one({"id": id, "status": Status.active}) - bot = BotSchema.model_validate(bot) - if not bot: + bot_model = BotTableCrud(session=session).get_one(bot_id=id) + if not bot_model: return json_response_message("No active bot found.") - else: - response = botModel.deactivate(bot) - if response: - return json_response_message( - "Active orders closed, sold base asset, deactivated" - ) - else: - return json_response_error("Error deactivating bot.") + + bot_instance = CreateDealController(bot_model, db_table=BotTable) + try: + bot_instance.close_all() + return json_response_message( + "Active orders closed, sold base asset, deactivated" + ) + except BinbotErrors as error: + bot_instance.controller.update_logs(bot_id=id, log_message=error.message) + return json_response_error(error.message) @bot_blueprint.post("/bot/errors/{bot_id}", tags=["bots"]) -def bot_errors(bot_id: str, bot_errors: ErrorsRequestBody): +def bot_errors( + bot_id: str, bot_errors: ErrorsRequestBody, session: Session = Depends(get_session) +): """ POST errors to a bot @@ -132,7 +152,7 @@ def bot_errors(bot_id: str, bot_errors: ErrorsRequestBody): request_body = bot_errors.model_dump(mode="python") bot_errors = request_body.get("errors", None) try: - Bot(collection_name="bots").post_errors_by_id(bot_id, bot_errors) + BotTableCrud(session=session).update_logs(bot_errors, bot_id=bot_id) except Exception as error: return json_response_error(f"Error posting errors: {error}") return json_response_message("Errors posted successfully.") diff --git a/api/database/api_db.py b/api/database/api_db.py index 4fcdd9581..a48ff6692 100644 --- a/api/database/api_db.py +++ b/api/database/api_db.py @@ -195,7 +195,7 @@ def create_dummy_bot(self): bot = BotTable( pair="BTCUSDT", balance_size_to_use="1", - balance_to_use=1, + fiat="USDC", base_order_size=15, deal_id=deal.id, cooldown=0, diff --git a/api/database/bot_crud.py b/api/database/bot_crud.py index b68ad1357..36b7ce4f3 100644 --- a/api/database/bot_crud.py +++ b/api/database/bot_crud.py @@ -3,13 +3,12 @@ from fastapi import Query from sqlmodel import Session, asc, desc, or_, select, case from time import time -from deals.controllers import CreateDealController from bots.schemas import BotSchema from database.models.bot_table import BotTable from database.models.deal_table import DealTable from database.utils import independent_session from deals.models import DealModel -from tools.enum_definitions import BinbotEnums, Status, Strategy +from tools.enum_definitions import BinbotEnums, Status from psycopg.types.json import Json @@ -48,7 +47,7 @@ def update_logs( if bot_id: bot_object: BotTable | BotSchema = self.session.get(BotTable, bot_id) elif not bot: - raise ValueError("Bot id or bot BotSchema object is required") + raise ValueError("Bot id or BotSchema | BotTable object is required") bot_object = bot @@ -59,6 +58,8 @@ def update_logs( current_logs.append(log_message) bot_object.logs = json.dumps(current_logs) + + # db operations self.session.add(bot_object) self.session.commit() self.session.close() @@ -66,7 +67,7 @@ def update_logs( def get( self, - status, + status: Status | None = None, start_date: float | None = None, end_date: float | None = None, no_cooldown=False, @@ -83,10 +84,8 @@ def get( """ statement = select(BotTable) - if status in BinbotEnums.statuses: + if status and status in BinbotEnums.statuses: statement.where(BotTable.status == status) - else: - raise ValueError("Invalid status") if start_date: statement.where(BotTable.created_at >= start_date) @@ -128,16 +127,28 @@ def get( return bots - def get_one(self, bot_id: str | None = None, symbol: str | None = None): + def get_one( + self, + bot_id: str | None = None, + symbol: str | None = None, + status: Status | None = None, + ): """ Get one bot by id or symbol """ if bot_id: bot = self.session.get(BotTable, bot_id) elif symbol: - bot = self.session.exec( - select(BotTable).where(BotTable.pair == symbol) - ).first() + if status: + bot = self.session.exec( + select(BotTable).where( + BotTable.pair == symbol, BotTable.status == status + ) + ).first() + else: + bot = self.session.exec( + select(BotTable).where(BotTable.pair == symbol) + ).first() else: raise ValueError("Invalid bot id or symbol") @@ -166,13 +177,16 @@ def create(self, data: BotSchema): self.session.close() return bot - def edit(self, id: str, data: BotSchema): + def save(self, data: BotTable): """ - Edit a bot + Save bot + + This can be an edit of an entire object + or just a few fields """ - bot = self.session.get(BotTable, id) + bot = self.session.get(BotTable, data.id) if not bot: - return bot + raise ValueError("Bot not found") # double check orders and deal are not overwritten dumped_bot = data.model_dump(exclude_unset=True) @@ -197,7 +211,8 @@ def delete(self, bot_ids: List[str] = Query(...)): def update_status(self, bot: BotTable, status: Status) -> BotTable: """ - Activate a bot by opening a deal + Mostly as a replacement of the previous "activate" and "deactivate" + although any Status can be passed now """ bot.status = status # db operations @@ -211,6 +226,8 @@ def update_status(self, bot: BotTable, status: Status) -> BotTable: def get_active_pairs(self): """ Get all active pairs + + a replacement of the previous "distinct pairs" query """ statement = select(BotTable.pair).where(BotTable.status == Status.active) pairs = self.session.exec(statement).all() diff --git a/api/database/models/bot_table.py b/api/database/models/bot_table.py index c7072384b..d7a0dd1f8 100644 --- a/api/database/models/bot_table.py +++ b/api/database/models/bot_table.py @@ -1,9 +1,12 @@ +import json from uuid import uuid4, UUID from time import time from typing import TYPE_CHECKING, List, Optional +from pydantic import Json, field_serializer, field_validator from sqlalchemy import JSON, Column, Enum from tools.enum_definitions import ( BinanceKlineIntervals, + BinbotEnums, CloseConditions, Status, Strategy, @@ -41,7 +44,7 @@ class BotTable(SQLModel, table=True): updated_at: float = Field(default_factory=lambda: time() * 1000) deal: Optional["DealTable"] = Relationship(back_populates="bot") dynamic_trailling: bool = Field(default=False) - logs: JSON = Field(default="[]", sa_column=Column(JSON)) + logs: List[Json[str]] = Field(default=[], sa_column=Column(JSON)) mode: str = Field(default="manual") name: str = Field(default="Default bot") # filled up internally @@ -61,5 +64,92 @@ class BotTable(SQLModel, table=True): short_sell_price: float = Field(default=0) total_commission: float = Field(default=0) - class Config: - arbitrary_types_allowed = True + model_config = { + "arbitrary_types_allowed": True, + "json_schema_extra": { + "description": "Most fields are optional. Deal field is generated internally, orders are filled up by Exchange", + "examples": [ + { + "pair": "BNBUSDT", + "fiat": "USDC", + "base_order_size": 15, + "candlestick_interval": "15m", + "cooldown": 0, + "logs": [], + # Manual is triggered by the terminal dashboard, autotrade by research app, + "mode": "manual", + "name": "Default bot", + "orders": [], + "status": "inactive", + "stop_loss": 0, + "take_profit": 2.3, + "trailling": "true", + "trailling_deviation": 0.63, + "trailling_profit": 2.3, + "strategy": "long", + "short_buy_price": 0, + "short_sell_price": 0, + "total_commission": 0, + } + ], + }, + } + + @field_validator("pair", "candlestick_interval") + @classmethod + def check_names_not_empty(cls, v): + assert v != "", "Empty pair field." + return v + + @field_validator("base_order_size") + @classmethod + def countables(cls, v): + if isinstance(v, float): + return v + elif isinstance(v, str): + return float(v) + elif isinstance(v, int): + return float(v) + else: + raise ValueError(f"{v} must be a number (float, int or string)") + + @field_validator( + "stop_loss", + "take_profit", + "trailling_deviation", + "trailling_profit", + mode="before", + ) + @classmethod + def check_percentage(cls, v): + if 0 <= float(v) < 100: + return v + else: + raise ValueError(f"{v} must be a percentage") + + @field_validator("mode") + @classmethod + def check_mode(cls, v: str): + if v not in BinbotEnums.mode: + raise ValueError(f'Status must be one of {", ".join(BinbotEnums.mode)}') + return v + + @field_validator("strategy") + @classmethod + def check_strategy(cls, v: str): + if v not in BinbotEnums.strategy: + raise ValueError(f'Status must be one of {", ".join(BinbotEnums.strategy)}') + return v + + @field_validator("trailling") + @classmethod + def booleans(cls, v: bool): + if isinstance(v, bool): + return v + else: + raise ValueError(f"{v} must be a boolean") + + @field_serializer("logs") + @classmethod + def logs_serializer(cls, v): + return json.loads(v) diff --git a/api/database/models/paper_trading_table.py b/api/database/models/paper_trading_table.py index 05ffcd777..64d29e285 100644 --- a/api/database/models/paper_trading_table.py +++ b/api/database/models/paper_trading_table.py @@ -1,9 +1,12 @@ +import json from uuid import uuid4, UUID from time import time from typing import TYPE_CHECKING, List, Optional +from pydantic import field_serializer, field_validator from sqlalchemy import JSON, Column, Enum from tools.enum_definitions import ( BinanceKlineIntervals, + BinbotEnums, CloseConditions, Status, Strategy, @@ -70,5 +73,92 @@ class PaperTradingTable(SQLModel, table=True): total_commission: float = Field(default=0) updated_at: float = Field(default_factory=lambda: time() * 1000) - class Config: - arbitrary_types_allowed = True + model_config = { + "arbitrary_types_allowed": True, + "json_schema_extra": { + "description": "Most fields are optional. Deal field is generated internally, orders are filled up by Exchange", + "examples": [ + { + "pair": "BNBUSDT", + "fiat": "USDC", + "base_order_size": 15, + "candlestick_interval": "15m", + "cooldown": 0, + "logs": [], + # Manual is triggered by the terminal dashboard, autotrade by research app, + "mode": "manual", + "name": "Default bot", + "orders": [], + "status": "inactive", + "stop_loss": 0, + "take_profit": 2.3, + "trailling": "true", + "trailling_deviation": 0.63, + "trailling_profit": 2.3, + "strategy": "long", + "short_buy_price": 0, + "short_sell_price": 0, + "total_commission": 0, + } + ], + }, + } + + @field_validator("pair", "candlestick_interval") + @classmethod + def check_names_not_empty(cls, v): + assert v != "", "Empty pair field." + return v + + @field_validator("base_order_size") + @classmethod + def countables(cls, v): + if isinstance(v, float): + return v + elif isinstance(v, str): + return float(v) + elif isinstance(v, int): + return float(v) + else: + raise ValueError(f"{v} must be a number (float, int or string)") + + @field_validator( + "stop_loss", + "take_profit", + "trailling_deviation", + "trailling_profit", + mode="before", + ) + @classmethod + def check_percentage(cls, v): + if 0 <= float(v) < 100: + return v + else: + raise ValueError(f"{v} must be a percentage") + + @field_validator("mode") + @classmethod + def check_mode(cls, v: str): + if v not in BinbotEnums.mode: + raise ValueError(f'Status must be one of {", ".join(BinbotEnums.mode)}') + return v + + @field_validator("strategy") + @classmethod + def check_strategy(cls, v: str): + if v not in BinbotEnums.strategy: + raise ValueError(f'Status must be one of {", ".join(BinbotEnums.strategy)}') + return v + + @field_validator("trailling") + @classmethod + def booleans(cls, v: bool): + if isinstance(v, bool): + return v + else: + raise ValueError(f"{v} must be a boolean") + + @field_serializer("logs") + @classmethod + def logs_serializer(cls, v): + return json.loads(v) diff --git a/api/database/paper_trading_crud.py b/api/database/paper_trading_crud.py index 8fdf328f1..36f700d94 100644 --- a/api/database/paper_trading_crud.py +++ b/api/database/paper_trading_crud.py @@ -1,9 +1,10 @@ from time import time -from sqlmodel import Session +from sqlmodel import Session, or_, select, case, desc, asc +from database.models.deal_table import DealTable from database.models.paper_trading_table import PaperTradingTable from database.utils import independent_session -from tools.enum_definitions import Status +from tools.enum_definitions import BinbotEnums, Status class PaperTradingTableCrud: @@ -48,30 +49,115 @@ def delete(self, id: str) -> bool: self.session.close() return True - def get(self, id: str) -> PaperTradingTable: + def get( + self, + status: Status | None = None, + start_date: float | None = None, + end_date: float | None = None, + no_cooldown=False, + limit: int = 200, + offset: int = 0, + ) -> PaperTradingTable: """ - Get a paper trading account by id + Get all bots in the db except archived + Args: + - status: Status enum + - start_date and end_date are timestamps in milliseconds + - no_cooldown: bool - filter out bots that are in cooldown + - limit and offset for pagination """ - paper_trading = self.session.get(PaperTradingTable, id) + statement = select(PaperTradingTable) + + if status and status in BinbotEnums.statuses: + statement.where(PaperTradingTable.status == status) + + if start_date: + statement.where(PaperTradingTable.created_at >= start_date) + + if end_date: + statement.where(PaperTradingTable.created_at <= end_date) + + if status and no_cooldown: + current_timestamp = time() + cooldown_condition = cooldown_condition = or_( + PaperTradingTable.status == status, + case( + ( + (DealTable.sell_timestamp > 0), + current_timestamp - DealTable.sell_timestamp + < (PaperTradingTable.cooldown * 1000), + ), + else_=( + current_timestamp - PaperTradingTable.created_at + < (PaperTradingTable.cooldown * 1000) + ), + ), + ) + + statement.where(cooldown_condition) + + # sorting + statement.order_by( + desc(PaperTradingTable.created_at), + case((PaperTradingTable.status == Status.active, 1), else_=2), + asc(PaperTradingTable.pair), + ) + + # pagination + statement.limit(limit).offset(offset) + + bots = self.session.exec(statement).all() self.session.close() - return paper_trading - def activate(self, paper_trading: PaperTradingTable) -> PaperTradingTable: + return bots + + def update_status( + self, paper_trading: PaperTradingTable, status: Status + ) -> PaperTradingTable: """ Activate a paper trading account """ - paper_trading.status = Status.active + paper_trading.status = status self.session.add(paper_trading) self.session.commit() self.session.close() return paper_trading - def deactivate(self, paper_trading: PaperTradingTable) -> PaperTradingTable: + def get_one( + self, + bot_id: str | None = None, + symbol: str | None = None, + status: Status | None = None, + ): """ - Deactivate a paper trading account + Get one bot by id or symbol """ - paper_trading.status = Status.inactive - self.session.add(paper_trading) - self.session.commit() + if bot_id: + bot = self.session.get(PaperTradingTable, bot_id) + elif symbol: + if status: + bot = self.session.exec( + select(PaperTradingTable).where( + PaperTradingTable.pair == symbol, + PaperTradingTable.status == status, + ) + ).first() + else: + bot = self.session.exec( + select(PaperTradingTable).where(PaperTradingTable.pair == symbol) + ).first() + else: + raise ValueError("Invalid bot id or symbol") + self.session.close() - return paper_trading + return bot + + def get_active_pairs(self): + """ + Get all active bots + """ + bots = self.session.exec( + select(PaperTradingTable).where(PaperTradingTable.status == Status.active) + ).all() + self.session.close() + return bots diff --git a/api/deals/base.py b/api/deals/base.py index b98621a4a..056c6b03a 100644 --- a/api/deals/base.py +++ b/api/deals/base.py @@ -1,11 +1,7 @@ from typing import Tuple -import uuid -from time import time from datetime import datetime - from database.bot_crud import BotTableCrud from database.paper_trading_crud import PaperTradingTableCrud -from database.models.paper_trading_table import PaperTradingTable from database.models.bot_table import BotTable from deals.models import BinanceOrderModel, DealModel from orders.controller import OrderController @@ -16,7 +12,7 @@ InsufficientBalance, MarginLoanNotFound, ) -from tools.enum_definitions import DealType, Status, Strategy +from tools.enum_definitions import DealType, OrderSide, Status, Strategy from base_producer import BaseProducer @@ -36,7 +32,7 @@ class BaseDeal(OrderController): def __init__(self, bot: BotTable, controller: PaperTradingTableCrud | BotTableCrud): self.active_bot = bot - self.controller: PaperTradingTableCrud | BotTable = controller + self.controller: PaperTradingTableCrud | BotTableCrud = controller() self.market_domination_reversal: bool | None = None self.price_precision = self.calculate_price_precision(bot.pair) self.qty_precision = self.calculate_qty_precision(bot.pair) @@ -52,13 +48,9 @@ def __repr__(self) -> str: """ return f"BaseDeal({self.__dict__})" - def generate_id(self): - return uuid.uuid4() - def compute_qty(self, pair): """ Helper function to compute buy_price. - Previous qty = bot.deal["buy_total_qty"] """ asset = self.find_baseAsset(pair) @@ -103,50 +95,6 @@ def compute_margin_buy_back(self) -> Tuple[float | int, float | int]: return qty, free - def simulate_order(self, pair, qty, side): - """ - Price is determined by market - to help trigger the order immediately - """ - price = float(self.matching_engine(pair, True, qty)) - order = { - "symbol": pair, - "orderId": self.generate_id().int, - "orderListId": -1, - "clientOrderId": self.generate_id().hex, - "transactTime": time() * 1000, - "price": price, - "origQty": qty, - "executedQty": qty, - "cummulativeQuoteQty": qty, - "status": "FILLED", - "timeInForce": "GTC", - "type": "LIMIT", - "side": side, - "fills": [], - } - return order - - def simulate_response_order(self, pair, qty, side): - price = float(self.matching_engine(pair, True, qty)) - response_order = { - "symbol": pair, - "orderId": self.generate_id().int, - "orderListId": -1, - "clientOrderId": self.generate_id().hex, - "transactTime": time() * 1000, - "price": price, - "origQty": qty, - "executedQty": qty, - "cummulativeQuoteQty": qty, - "status": "FILLED", - "timeInForce": "GTC", - "type": "LIMIT", - "side": side, - "fills": [], - } - return response_order - def replace_order(self, cancel_order_id): payload = { "symbol": self.active_bot.pair, @@ -177,9 +125,9 @@ def close_open_orders(self, symbol): payload={"symbol": symbol, "orderId": order["orderId"]}, ) for order in self.active_bot.orders: - if order.order_id == order["orderId"]: + if order.id == order["orderId"]: self.active_bot.orders.remove(order) - self.active_bot.errors.append( + self.controller.update_logs( "base_order not executed, therefore cancelled" ) self.active_bot.status = Status.error @@ -206,73 +154,6 @@ def verify_deal_close_order(self): return None - def base_order(self): - """ - Required initial order to trigger long strategy bot. - Other orders require this to execute, - therefore should fail if not successful - - 1. Initial base purchase - 2. Set take_profit - """ - - # Long position does not need qty in take_profit - # initial price with 1 qty should return first match - price = float(self.matching_engine(self.active_bot.pair, True)) - qty = round_numbers( - (float(self.active_bot.base_order_size) / float(price)), - self.qty_precision, - ) - # setup stop_loss_price - stop_loss_price: float = 0 - if float(self.active_bot.stop_loss) > 0: - stop_loss_price = price - (price * (float(self.active_bot.stop_loss) / 100)) - - if self.controller == PaperTradingTable: - res = self.simulate_order( - self.active_bot.pair, - qty, - "BUY", - ) - else: - res = self.buy_order( - symbol=self.active_bot.pair, - qty=qty, - price=supress_notation(price, self.price_precision), - ) - - order_data = BinanceOrderModel( - timestamp=res["transactTime"], - order_id=res["orderId"], - deal_type=DealType.base_order, - pair=res["symbol"], - order_side=res["side"], - order_type=res["type"], - price=res["price"], - qty=res["origQty"], - time_in_force=res["timeInForce"], - status=res["status"], - ) - - self.active_bot.orders.append(order_data) - tp_price = float(res["price"]) * 1 + (float(self.active_bot.take_profit) / 100) - - self.active_bot.deal = DealModel( - buy_timestamp=res["transactTime"], - buy_price=res["price"], - buy_total_qty=res["origQty"], - current_price=res["price"], - take_profit_price=tp_price, - stop_loss_price=stop_loss_price, - ) - - # Activate bot - document = self.controller.activate(self.active_bot) - # do this after db operations in case there is rollback - # avoids sending unnecessary signals - self.base_producer.update_required(self.producer, "ACTIVATE_BOT") - return document - def margin_liquidation(self, pair: str): """ Emulate Binance Dashboard One click liquidation function diff --git a/api/deals/controllers.py b/api/deals/controllers.py index f48c9ee60..f16092a84 100644 --- a/api/deals/controllers.py +++ b/api/deals/controllers.py @@ -6,15 +6,13 @@ from bots.schemas import BotSchema from deals.base import BaseDeal from deals.margin import MarginDeal -from deals.models import BinanceOrderModel -from pymongo import ReturnDocument -from tools.enum_definitions import DealType, Status, Strategy +from deals.models import BinanceOrderModel, DealModel +from tools.enum_definitions import DealType, OrderSide, Status, Strategy from tools.exceptions import TakeProfitError from tools.handle_error import ( - encode_json, handle_binance_errors, ) -from tools.round_numbers import round_numbers +from tools.round_numbers import round_numbers, supress_notation class CreateDealController(BaseDeal): @@ -46,7 +44,7 @@ def __init__( self.active_bot = bot self.db_table = db_table - def compute_qty(self, pair): + def compute_qty(self, pair: str) -> float | None: """ Helper function to compute buy_price. Previous qty = bot.deal["buy_total_qty"] @@ -59,6 +57,72 @@ def compute_qty(self, pair): qty = round_numbers(balance[0], self.qty_precision) return qty + def base_order(self): + """ + Required initial order to trigger long strategy bot. + Other orders require this to execute, + therefore should fail if not successful + + 1. Initial base purchase + 2. Set take_profit + """ + + # Long position does not need qty in take_profit + # initial price with 1 qty should return first match + price = float(self.matching_engine(self.active_bot.pair, True)) + qty = round_numbers( + (float(self.active_bot.base_order_size) / float(price)), + self.qty_precision, + ) + # setup stop_loss_price + stop_loss_price = 0 + if float(self.active_bot.stop_loss) > 0: + stop_loss_price = price - (price * (float(self.active_bot.stop_loss) / 100)) + + if self.controller == PaperTradingTableCrud: + res = self.simulate_order( + self.active_bot.pair, + OrderSide.buy, + ) + else: + res = self.buy_order( + symbol=self.active_bot.pair, + qty=qty, + price=supress_notation(price, self.price_precision), + ) + + order_data = BinanceOrderModel( + timestamp=res["transactTime"], + order_id=res["orderId"], + deal_type=DealType.base_order, + pair=res["symbol"], + order_side=res["side"], + order_type=res["type"], + price=res["price"], + qty=res["origQty"], + time_in_force=res["timeInForce"], + status=res["status"], + ) + + self.active_bot.orders.append(order_data) + tp_price = float(res["price"]) * 1 + (float(self.active_bot.take_profit) / 100) + + self.active_bot.deal = DealModel( + buy_timestamp=res["transactTime"], + buy_price=res["price"], + buy_total_qty=res["origQty"], + current_price=res["price"], + take_profit_price=tp_price, + stop_loss_price=stop_loss_price, + ) + + # Activate bot + document = self.open_deal(self.active_bot) + # do this after db operations in case there is rollback + # avoids sending unnecessary signals + self.base_producer.update_required(self.producer, "ACTIVATE_BOT") + return document + def take_profit_order(self) -> BotSchema: """ take profit order (Binance take_profit) @@ -79,22 +143,7 @@ def take_profit_order(self) -> BotSchema: price = round_numbers(price, self.price_precision) if self.db_table == "paper_trading": - res = self.simulate_order(self.active_bot.pair, qty, "SELL") - if price: - res = self.simulate_order( - self.active_bot.pair, - qty, - "SELL", - ) - else: - price = (1 + (float(self.active_bot.take_profit) / 100)) * float( - deal_buy_price - ) - res = self.simulate_order( - self.active_bot.pair, - qty, - "SELL", - ) + res = self.simulate_order(self.active_bot.pair, OrderSide.sell) else: qty = round_numbers(qty, self.qty_precision) price = round_numbers(price, self.price_precision) @@ -127,23 +176,9 @@ def take_profit_order(self) -> BotSchema: self.active_bot.deal.sell_qty = res["origQty"] self.active_bot.deal.sell_timestamp = res["transactTime"] self.active_bot.status = Status.completed - msg = "Completed take profit" - self.active_bot.errors.append(msg) - - try: - bot = encode_json(self.active_bot) - if "_id" in bot: - bot.pop("_id") - - bot = self.db_collection.find_one_and_update( - {"id": self.active_bot.id}, - { - "$set": bot, - }, - return_document=ReturnDocument.AFTER, - ) - except Exception as error: - raise TakeProfitError(error) + + bot = self.controller.save(self.active_bot) + self.controller.update_logs("Completed take profit", self.active_bot) return bot @@ -160,7 +195,7 @@ def close_all(self) -> None: if len(orders) > 0: for d in orders: if d.status == "NEW" or d.status == "PARTIALLY_FILLED": - self.update_deal_logs( + self.controller.update_logs( "Failed to close all active orders (status NEW), retrying...", self.active_bot, ) @@ -178,7 +213,7 @@ def close_all(self) -> None: return - def update_take_profit(self, order_id) -> None: + def update_take_profit(self, order_id: int) -> BotTable: """ Update take profit after websocket order endpoint triggered - Close current opened take profit order @@ -187,9 +222,7 @@ def update_take_profit(self, order_id) -> None: """ bot = self.active_bot if bot.deal: - find_base_order = next( - (order.order_id == order_id for order in bot.orders), None - ) + find_base_order = next((order.id == order_id for order in bot.orders), None) if find_base_order: so_deal_price = bot.deal.buy_price # Create new take profit order @@ -237,25 +270,27 @@ def update_take_profit(self, order_id) -> None: new_deals.append(take_profit_order) self.active_bot.orders = new_deals self.active_bot.total_commission = total_commission - self.active_bot.errors.append("take_profit deal successfully updated") - self.db.bots.update_one( - {"id": self.active_bot.id}, - {"$set": self.active_bot.model_dump()}, - ) - return + self.controller.save(self.active_bot) + self.controller.update_logs("take_profit deal successfully updated") + return self.active_bot else: - self.update_deal_logs( + self.controller.update_logs( "Error: Bot does not contain a base order deal", self.active_bot ) + raise ValueError("Bot does not contain a base order deal") - def open_deal(self) -> None: + def open_deal(self) -> BotTable: """ - Mandatory deals section + Bot activation requires: + + 1. Opening a new deal, which entails opening orders + 2. Updating stop loss and take profit + 3. Updating trailling + 4. Save in db - - If base order deal is not executed, bot is not activated + - If bot DOES have a base order, we still need to update stop loss and take profit and trailling """ - # If there is already a base order do not execute base_order_deal = next( ( bo_deal @@ -272,7 +307,7 @@ def open_deal(self) -> None: ).margin_short_base_order() else: bot = self.base_order() - self.active_bot = BotSchema(**bot) + self.active_bot = BotTable.model_validate(bot) """ Optional deals section @@ -282,10 +317,14 @@ def open_deal(self) -> None: # Update stop loss regarless of base order if float(self.active_bot.stop_loss) > 0: - if self.active_bot.strategy == Strategy.margin_short: - self.active_bot = MarginDeal( - bot=self.active_bot, db_collection_name=self.db_table - ).set_margin_short_stop_loss() + if ( + self.active_bot.strategy == Strategy.margin_short + and self.active_bot.stop_loss > 0 + ): + price = self.active_bot.deal.margin_short_sell_price + self.active_bot.deal.stop_loss_price = price + ( + price * (float(self.active_bot.stop_loss) / 100) + ) else: buy_price = float(self.active_bot.deal.buy_price) stop_loss_price = buy_price - ( @@ -297,16 +336,19 @@ def open_deal(self) -> None: # Margin short Take profit if ( - float(self.active_bot.take_profit) > 0 + self.active_bot.take_profit > 0 and self.active_bot.strategy == Strategy.margin_short ): - self.active_bot = MarginDeal( - bot=self.active_bot, db_collection_name=self.db_collection.name - ).set_margin_take_profit() + if self.active_bot.take_profit: + price = float(self.active_bot.deal.margin_short_sell_price) + take_profit_price = price - ( + price * (self.active_bot.take_profit) / 100 + ) + self.active_bot.deal.take_profit_price = take_profit_price # Keep trailling_stop_loss_price up to date in case of failure to update in autotrade # if we don't do this, the trailling stop loss will trigger - if self.active_bot.deal and ( + if ( self.active_bot.deal.trailling_stop_loss_price > 0 or self.active_bot.deal.trailling_stop_loss_price < self.active_bot.deal.buy_price @@ -319,9 +361,6 @@ def open_deal(self) -> None: self.active_bot.deal.trailling_stop_loss_price = 0 self.active_bot.status = Status.active - bot = self.active_bot.model_dump() - if "_id" in bot: - bot.pop("_id") - - self.db_collection.update_one({"id": self.active_bot.id}, {"$set": bot}) - return + bot = self.controller.save(self.active_bot) + self.controller.update_logs("Bot activated", bot) + return bot diff --git a/api/deals/margin.py b/api/deals/margin.py index d8e49ee20..d370b1622 100644 --- a/api/deals/margin.py +++ b/api/deals/margin.py @@ -1,9 +1,10 @@ import logging -from time import time from urllib.error import HTTPError +from database.bot_crud import BotTableCrud +from database.models.bot_table import BotTable +from database.paper_trading_crud import PaperTradingTableCrud from deals.models import BinanceOrderModel -from base_producer import BaseProducer -from tools.enum_definitions import CloseConditions, DealType, Strategy +from tools.enum_definitions import CloseConditions, DealType, OrderSide, Strategy from bots.schemas import BotSchema from tools.enum_definitions import Status from deals.base import BaseDeal @@ -12,35 +13,12 @@ class MarginDeal(BaseDeal): - def __init__(self, bot, db_collection_name) -> None: + def __init__( + self, bot: BotTable, controller: PaperTradingTableCrud | BotTableCrud + ) -> None: self.active_bot: BotSchema # Inherit from parent class - super().__init__(bot, db_collection_name=db_collection_name) - self.base_producer = BaseProducer() - self.producer = self.base_producer.start_producer() - - def simulate_margin_order(self, qty, side): - price = float(self.matching_engine(self.active_bot.pair, True, qty)) - order = { - "symbol": self.active_bot.pair, - "orderId": self.generate_id().int, - "orderListId": -1, - "clientOrderId": self.generate_id().hex, - "transactTime": time() * 1000, - "price": price, - "origQty": qty, - "executedQty": qty, - "cummulativeQuoteQty": qty, - "status": "FILLED", - "timeInForce": "GTC", - "type": "LIMIT", - "side": side, - "marginBuyBorrowAmount": 5, - "marginBuyBorrowAsset": "BTC", - "isIsolated": "true", - "fills": [], - } - return order + super().__init__(bot, controller=controller) def get_remaining_assets(self) -> tuple[float, float]: """ @@ -51,14 +29,14 @@ def get_remaining_assets(self) -> tuple[float, float]: """ if float(self.isolated_balance[0]["quoteAsset"]["borrowed"]) > 0: - self.update_deal_logs( + self.controller.update_logs( f'Borrowed {self.isolated_balance[0]["quoteAsset"]["asset"]} still remaining, please clear out manually', self.active_bot, ) self.active_bot.status = Status.error if float(self.isolated_balance[0]["baseAsset"]["borrowed"]) > 0: - self.update_deal_logs( + self.controller.update_logs( f'Borrowed {self.isolated_balance[0]["baseAsset"]["asset"]} still remaining, please clear out manually', self.active_bot, ) @@ -71,7 +49,7 @@ def get_remaining_assets(self) -> tuple[float, float]: base_asset, self.qty_precision ) - def cancel_open_orders(self, deal_type): + def cancel_open_orders(self, deal_type: DealType) -> None: """ Given an order deal_type i.e. take_profit, stop_loss etc cancel currently open orders to unblock funds @@ -88,11 +66,11 @@ def cancel_open_orders(self, deal_type): try: # First cancel old order to unlock balance self.cancel_margin_order(symbol=self.active_bot.pair, order_id=order_id) - self.update_deal_logs( + self.controller.update_logs( "Old take profit order cancelled", self.active_bot ) except HTTPError: - self.update_deal_logs( + self.controller.update_logs( "Take profit order not found, no need to cancel", self.active_bot ) return @@ -104,7 +82,7 @@ def cancel_open_orders(self, deal_type): return - def terminate_failed_transactions(self): + def terminate_failed_transactions(self) -> None: """ Transfer back from isolated account to spot account Disable isolated pair (so we don't reach the limit) @@ -117,7 +95,7 @@ def terminate_failed_transactions(self): amount=qty, ) - def init_margin_short(self, initial_price): + def init_margin_short(self, initial_price: float) -> None: """ Pre-tasks for db_collection = bots These tasks are not necessary for paper_trading @@ -126,7 +104,9 @@ def init_margin_short(self, initial_price): 2. create loan with qty given by market 3. borrow 2.5x to do base order """ - self.update_deal_logs("Initializating margin_short tasks", self.active_bot) + self.controller.update_logs( + "Initializating margin_short tasks", self.active_bot + ) # Check margin account balance first balance = float(self.isolated_balance[0]["quoteAsset"]["free"]) asset = self.active_bot.pair.replace(self.active_bot.balance_to_use, "") @@ -137,9 +117,9 @@ def init_margin_short(self, initial_price): asset=asset, isolated_symbol=self.active_bot.pair ) error_msg = f"Checking borrowable amount: {borrow_res['amount']} (amount), {borrow_res['borrowLimit']} (limit)" - self.update_deal_logs(error_msg, self.active_bot) + self.controller.update_logs(error_msg, self.active_bot) except BinanceErrors as error: - self.update_deal_logs(error.message, self.active_bot) + self.controller.update_logs(error.message, self.active_bot) if error.code == -11001 or error.code == -3052: # Isolated margin account needs to be activated with a transfer self.transfer_spot_to_isolated_margin( @@ -197,7 +177,7 @@ def init_margin_short(self, initial_price): return - def terminate_margin_short(self, buy_back_fiat: bool = True): + def terminate_margin_short(self, buy_back_fiat: bool = True) -> BotTable: """ Args: @@ -246,7 +226,7 @@ def terminate_margin_short(self, buy_back_fiat: bool = True): # false alarm pass except Exception as error: - self.update_deal_logs(error, self.active_bot) + self.controller.update_logs(error, self.active_bot) # Continue despite errors to avoid losses # most likely it is still possible to update bot pass @@ -308,7 +288,7 @@ def terminate_margin_short(self, buy_back_fiat: bool = True): self.active_bot.errors.append("Loan not found for this bot.") # Save in two steps, because it takes time for Binance to process repayments - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) try: # get new balance @@ -330,16 +310,18 @@ def terminate_margin_short(self, buy_back_fiat: bool = True): error_msg = f"Failed to transfer isolated assets to spot: {error}" logging.error(error_msg) self.active_bot.errors.append(error_msg) - return + return self.active_bot - completion_msg = f"{self.active_bot.pair} ISOLATED margin funds transferred back to SPOT." self.active_bot.status = Status.completed - self.active_bot.errors.append(completion_msg) + self.controller.update_logs( + f"{self.active_bot.pair} ISOLATED margin funds transferred back to SPOT.", + self.active_bot, + ) - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) return self.active_bot - def margin_short_base_order(self): + def margin_short_base_order(self) -> BotTable: """ Same functionality as usual base_order with a few more fields. This is used during open_deal @@ -349,21 +331,16 @@ def margin_short_base_order(self): """ initial_price = float(self.matching_engine(self.active_bot.pair, False)) - if self.db_collection.name == "bots": + if self.controller == BotTableCrud: self.init_margin_short(initial_price) - try: - order_res = self.sell_margin_order( - symbol=self.active_bot.pair, - qty=self.active_bot.deal.margin_short_base_order, - ) - except BinanceErrors as error: - if error.code == -3052: - print(error) - return + order_res = self.sell_margin_order( + symbol=self.active_bot.pair, + qty=self.active_bot.deal.margin_short_base_order, + ) else: - # Simulate Margin sell - # qty doesn't matter in paper bots - order_res = self.simulate_margin_order(1, "SELL") + order_res = self.simulate_margin_order( + pair=self.active_bot, side=OrderSide.sell + ) order_data = BinanceOrderModel( timestamp=order_res["transactTime"], @@ -391,9 +368,10 @@ def margin_short_base_order(self): # Activate bot self.active_bot.status = Status.active + self.active_bot = self.controller.save(self.active_bot) return self.active_bot - def streaming_updates(self, close_price: str): + def streaming_updates(self, close_price: str) -> None: """ Margin_short streaming updates """ @@ -418,14 +396,20 @@ def streaming_updates(self, close_price: str): ) * float(self.active_bot.deal.hourly_interest_rate) # bugs, normally this should be set at deal opening - if self.active_bot.deal.take_profit_price == 0: - self.set_margin_take_profit() + if ( + self.active_bot.deal.take_profit_price == 0 + and self.active_bot.take_profit > 0 + ): + price = self.active_bot.deal.margin_short_sell_price + self.active_bot.deal.take_profit_price = price - ( + price * (float(self.active_bot.take_profit) / 100) + ) logging.debug( f"margin_short streaming updating {self.active_bot.pair} @ {self.active_bot.deal.stop_loss_price} and interests {self.active_bot.deal.margin_short_loan_interest}" ) - self.save_bot_streaming(self.active_bot) + self.controller.save(self.active_bot) # Direction 1.1: downward trend (short) # Breaking trailling @@ -438,11 +422,11 @@ def streaming_updates(self, close_price: str): self.active_bot.trailling == "true" or self.active_bot.trailling ) and self.active_bot.deal.margin_short_sell_price > 0: self.update_trailling_profit(price) - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) else: # Execute the usual non-trailling take_profit - self.update_deal_logs( + self.controller.update_logs( f"Executing margin_short take_profit after hitting take_profit_price {self.active_bot.deal.stop_loss_price}", self.active_bot, ) @@ -457,7 +441,7 @@ def streaming_updates(self, close_price: str): if float(self.active_bot.deal.trailling_stop_loss_price) > 0 and float( close_price ) >= float(self.active_bot.deal.trailling_stop_loss_price): - self.update_deal_logs( + self.controller.update_logs( f"Hit trailling_stop_loss_price {self.active_bot.deal.trailling_stop_loss_price}. Selling {self.active_bot.pair}", self.active_bot, ) @@ -489,22 +473,7 @@ def streaming_updates(self, close_price: str): return - def set_margin_short_stop_loss(self): - """ - Sets stop_loss for margin_short at initial activation - """ - price = float(self.active_bot.deal.margin_short_sell_price) - if ( - hasattr(self.active_bot, "stop_loss") - and float(self.active_bot.stop_loss) > 0 - ): - self.active_bot.deal.stop_loss_price = price + ( - price * (float(self.active_bot.stop_loss) / 100) - ) - - return self.active_bot - - def set_margin_take_profit(self): + def set_margin_take_profit(self) -> None: """ Sets take_profit for margin_short at initial activation """ @@ -520,14 +489,16 @@ def set_margin_take_profit(self): return self.active_bot - def execute_stop_loss(self): + def execute_stop_loss(self) -> None: """ Execute stop loss when price is hit This is used during streaming updates """ # Margin buy (buy back) - if self.db_collection.name == "paper_trading": - res = self.simulate_margin_order(self.active_bot.deal.buy_total_qty, "BUY") + if self.controller == PaperTradingTableCrud: + res = self.simulate_margin_order( + self.active_bot.deal.buy_total_qty, OrderSide.buy + ) else: # Cancel orders first # paper_trading doesn't have real orders so no need to check @@ -562,11 +533,11 @@ def execute_stop_loss(self): msg = "Completed Stop loss order" self.active_bot.errors.append(msg) self.active_bot.status = Status.completed - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) return - def execute_take_profit(self): + def execute_take_profit(self) -> None: """ Execute take profit when price is hit. This can be a simple take_profit order when take_profit_price is hit or @@ -578,12 +549,14 @@ def execute_take_profit(self): - Buy back asset sold """ - if self.db_collection.name == "bots": + if self.controller == BotTableCrud: self.cancel_open_orders("take_profit") # Margin buy (buy back) - if self.db_collection.name == "paper_trading": - res = self.simulate_margin_order(self.active_bot.deal.buy_total_qty, "BUY") + if self.controller == PaperTradingTableCrud: + res = self.simulate_margin_order( + self.active_bot.deal.buy_total_qty, OrderSide.buy + ) else: res = self.margin_liquidation(self.active_bot.pair) @@ -618,11 +591,11 @@ def execute_take_profit(self): self.active_bot.errors.append(msg) self.active_bot.status = Status.completed - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) return - def switch_to_long_bot(self): + def switch_to_long_bot(self) -> BotTable: """ Switch to long strategy. Doing some parts of open_deal from scratch @@ -634,21 +607,21 @@ def switch_to_long_bot(self): 2. Calculate take_profit_price and stop_loss_price as usual 3. Create deal """ - self.update_deal_logs( + self.controller.update_logs( "Switching margin_short to long strategy", self.active_bot ) self.active_bot.strategy = Strategy.long - self.active_bot = self.create_new_bot_streaming(active_bot=self.active_bot) + self.controller.save(self.active_bot) bot = self.base_order() - self.active_bot = BotSchema(**bot) + self.active_bot = BotSchema.model_validate(bot) # Keep bot up to date in the DB # this avoid unsyched bots when errors ocurr in other functions - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) return self.active_bot - def update_trailling_profit(self, close_price): + def update_trailling_profit(self, close_price: float) -> None: # Fix potential bugs in bot updates if self.active_bot.deal.take_profit_price == 0: self.margin_short_base_order() @@ -669,8 +642,8 @@ def update_trailling_profit(self, close_price): self.active_bot.deal.trailling_stop_loss_price = ( stop_loss_trailling_price ) - self.active_bot = self.save_bot_streaming(self.active_bot) - self.update_deal_logs( + self.controller.save(self.active_bot) + self.controller.update_logs( f"{self.active_bot.pair} Setting trailling_stop_loss (short) and saved to DB", self.active_bot, ) @@ -699,7 +672,7 @@ def update_trailling_profit(self, close_price): # Reset stop_loss_price to avoid confusion in front-end self.active_bot.deal.stop_loss_price = 0 - self.update_deal_logs( + self.controller.update_logs( f"{self.active_bot.pair} Updating after broken first trailling_profit (short)", self.active_bot, ) @@ -725,7 +698,7 @@ def update_trailling_profit(self, close_price): # Update trailling_stop_loss self.active_bot.deal.trailling_stop_loss_price = new_trailling_stop_loss - def close_conditions(self, current_price): + def close_conditions(self, current_price: float) -> None: """ Check if there is a market reversal @@ -738,7 +711,7 @@ def close_conditions(self, current_price): self.market_domination_reversal and current_price > self.active_bot.deal.buy_price ): - self.update_deal_logs( + self.controller.update_logs( f"Closing bot according to close_condition: {self.active_bot.close_condition}", self.active_bot, ) diff --git a/api/deals/spot.py b/api/deals/spot.py index 2148c698a..1ce547651 100644 --- a/api/deals/spot.py +++ b/api/deals/spot.py @@ -1,10 +1,17 @@ import logging - -from base_producer import BaseProducer +from database.models.bot_table import BotTable +from database.paper_trading_crud import PaperTradingTableCrud +from database.schemas import Order from deals.base import BaseDeal from deals.margin import MarginDeal from deals.models import BinanceOrderModel -from tools.enum_definitions import CloseConditions, DealType, Status, Strategy +from tools.enum_definitions import ( + CloseConditions, + DealType, + OrderSide, + Status, + Strategy, +) from bots.schemas import BotSchema @@ -18,8 +25,6 @@ def __init__(self, bot, db_collection_name: str) -> None: # Inherit from parent class self.db_collection_name = db_collection_name super().__init__(bot, db_collection_name) - self.base_producer = BaseProducer() - self.producer = self.base_producer.start_producer() self.active_bot: BotSchema def switch_margin_short(self): @@ -34,20 +39,20 @@ def switch_margin_short(self): 2. Calculate take_profit_price and stop_loss_price as usual 3. Create deal """ - self.update_deal_logs( + self.controller.update_logs( "Resetting bot for margin_short strategy...", self.active_bot ) self.active_bot.strategy = Strategy.margin_short - self.active_bot = self.create_new_bot_streaming(active_bot=self.active_bot) + self.active_bot = self.controller.create(data=self.active_bot) self.active_bot = MarginDeal( bot=self.active_bot, db_collection_name=self.db_collection_name ).margin_short_base_order() - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) return self.active_bot - def execute_stop_loss(self): + def execute_stop_loss(self) -> BotTable: """ Update stop limit after websocket @@ -55,8 +60,8 @@ def execute_stop_loss(self): - Close current opened take profit order - Deactivate bot """ - self.update_deal_logs("Executing stop loss...", self.active_bot) - if self.db_collection.name == "paper_trading": + self.controller.update_logs("Executing stop loss...", self.active_bot) + if self.controller == PaperTradingTableCrud: qty = self.active_bot.deal.buy_total_qty else: qty = self.compute_qty(self.active_bot.pair) @@ -68,26 +73,28 @@ def execute_stop_loss(self): if not closed_orders: order = self.verify_deal_close_order() if order: - self.active_bot.errors.append( - "Execute stop loss previous order found! Appending..." + self.controller.update_logs( + "Execute stop loss previous order found! Appending...", + self.active_bot, ) self.active_bot.orders.append(order) else: - self.update_deal_logs( + self.controller.update_logs( "No quantity in balance, no closed orders. Cannot execute update stop limit.", self.active_bot, ) self.active_bot.status = Status.error - self.active_bot = self.save_bot_streaming(self.active_bot) - return + self.active_bot = self.controller.save(self.active_bot) + return self.active_bot # Dispatch fake order - if self.db_collection.name == "paper_trading": - res = self.simulate_order(self.active_bot.pair, qty, "SELL") + if self.controller == PaperTradingTableCrud: + res = self.simulate_order(pair=self.active_bot.pair, side=OrderSide.sell) else: - self.active_bot.errors.append( - "Dispatching sell order for trailling profit..." + self.controller.update_logs( + "Dispatching sell order for trailling profit...", + self.active_bot, ) # Dispatch real order res = self.sell_order(symbol=self.active_bot.pair, qty=qty) @@ -113,13 +120,14 @@ def execute_stop_loss(self): self.active_bot.deal.sell_price = res["price"] self.active_bot.deal.sell_qty = res["origQty"] self.active_bot.deal.sell_timestamp = res["transactTime"] - msg = "Completed Stop loss. " + msg = "Completed Stop loss." if self.active_bot.margin_short_reversal: - msg += "Scheduled to switch strategy" - self.active_bot.errors.append(msg) + msg += " Scheduled to switch strategy" + + self.controller.update_logs(msg) self.active_bot.status = Status.completed + self.active_bot = self.controller.save(self.active_bot) - self.active_bot = self.save_bot_streaming(self.active_bot) return self.active_bot def trailling_profit(self) -> BotSchema | None: @@ -127,7 +135,7 @@ def trailling_profit(self) -> BotSchema | None: Sell at take_profit price, because prices will not reach trailling """ - if self.db_collection.name == "paper_trading": + if self.controller == PaperTradingTableCrud: qty = self.active_bot.deal.buy_total_qty else: qty = self.compute_qty(self.active_bot.pair) @@ -137,30 +145,31 @@ def trailling_profit(self) -> BotSchema | None: if not closed_orders: order = self.verify_deal_close_order() if order: - self.active_bot.errors.append( - "Execute trailling profit previous order found! Appending..." + self.controller.update_logs( + "Execute trailling profit previous order found! Appending...", + self.active_bot, ) self.active_bot.orders.append(order) else: - self.update_deal_logs( + self.controller.update_logs( "No quantity in balance, no closed orders. Cannot execute update trailling profit.", self.active_bot, ) self.active_bot.status = Status.error - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) return self.active_bot # Dispatch fake order - if self.db_collection.name == "paper_trading": + if self.controller == PaperTradingTableCrud: res = self.simulate_order( self.active_bot.pair, - qty, - "SELL", + OrderSide.sell, ) else: - self.active_bot.errors.append( - "Dispatching sell order for trailling profit..." + self.controller.update_logs( + "Dispatching sell order for trailling profit...", + self.active_bot, ) # Dispatch real order # No price means market order @@ -194,10 +203,10 @@ def trailling_profit(self) -> BotSchema | None: self.active_bot.deal.sell_qty = res["origQty"] self.active_bot.deal.sell_timestamp = res["transactTime"] self.active_bot.status = Status.completed - msg = f"Completed take profit after failing to break trailling {self.active_bot.pair}" - self.active_bot.errors.append(msg) - - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) + self.controller.update_logs( + f"Completed take profit after failing to break trailling {self.active_bot.pair}" + ) return self.active_bot def streaming_updates(self, close_price, open_price): @@ -205,12 +214,13 @@ def streaming_updates(self, close_price, open_price): self.close_conditions(float(close_price)) self.active_bot.deal.current_price = close_price - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) # Stop loss - if float(self.active_bot.stop_loss) > 0 and float( - self.active_bot.deal.stop_loss_price - ) > float(close_price): + if ( + self.active_bot.stop_loss > 0 + and self.active_bot.deal.stop_loss_price > float(close_price) + ): self.execute_stop_loss() self.base_producer.update_required(self.producer, "EXECUTE_SPOT_STOP_LOSS") if self.active_bot.margin_short_reversal: @@ -218,7 +228,7 @@ def streaming_updates(self, close_price, open_price): self.base_producer.update_required( self.producer, "EXECUTE_SWITCH_MARGIN_SHORT" ) - self.update_deal_logs( + self.controller.update_logs( "Completed switch to margin short bot", self.active_bot ) @@ -265,11 +275,11 @@ def streaming_updates(self, close_price, open_price): new_trailling_stop_loss ) - self.update_deal_logs( + self.controller.update_logs( f"Updated {self.active_bot.pair} trailling_stop_loss_price {self.active_bot.deal.trailling_stop_loss_price}", self.active_bot, ) - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) # Direction 2 (downward): breaking the trailling_stop_loss # Make sure it's red candlestick, to avoid slippage loss @@ -282,14 +292,11 @@ def streaming_updates(self, close_price, open_price): # Red candlestick and (float(open_price) > float(close_price)) ): - self.update_deal_logs( + self.controller.update_logs( f"Hit trailling_stop_loss_price {self.active_bot.deal.trailling_stop_loss_price}. Selling {self.active_bot.pair}", self.active_bot, ) self.trailling_profit() - self.base_producer.update_required( - self.producer, "EXECUTE_SPOT_TRAILLING_PROFIT" - ) # Update unfilled orders unupdated_order = next( @@ -312,7 +319,11 @@ def streaming_updates(self, close_price, open_price): self.active_bot.orders[i].qty = order_response["origQty"] self.active_bot.orders[i].status = order_response["status"] - self.active_bot = self.save_bot_streaming(self.active_bot) + self.active_bot = self.controller.save(self.active_bot) + + self.base_producer.update_required( + self.producer, "EXECUTE_SPOT_STREAMING_UPDATES" + ) def close_conditions(self, current_price): """ diff --git a/api/mypy.ini b/api/mypy.ini index c6cf4fad0..9f41fb065 100644 --- a/api/mypy.ini +++ b/api/mypy.ini @@ -1,4 +1,4 @@ -[mypy] +[mypy.api] implicit_optional = True ignore_missing_imports = True plugins = pydantic.mypy @@ -8,4 +8,8 @@ warn_redundant_casts = True warn_unused_ignores = True check_untyped_defs = True implicit_reexport = True -explicit_package_bases = True \ No newline at end of file +explicit_package_bases = True +disallow_any_expr = True +disallow_untyped_calls = True +disallow_untyped_defs = True +check_untyped_defs = True diff --git a/api/orders/controller.py b/api/orders/controller.py index c1849b7aa..9841f812d 100644 --- a/api/orders/controller.py +++ b/api/orders/controller.py @@ -1,16 +1,20 @@ +from time import time +from uuid import uuid4 from account.account import Account -from api.database.bot_crud import BotTableCrud from tools.exceptions import DeleteOrderError from tools.enum_definitions import OrderType, TimeInForce, OrderSide from tools.handle_error import json_response, json_response_message -from tools.round_numbers import supress_notation +from tools.round_numbers import supress_notation, zero_remainder -class OrderController(BotTableCrud, Account): +class OrderController(Account): """ Always GTC and limit orders limit/market orders will be decided by matching_engine PRICE_FILTER decimals + + Methods and attributes here are all unrelated to database operations + this is highly tied to the Binance API """ def __init__(self) -> None: @@ -20,14 +24,84 @@ def __init__(self) -> None: self.qty_precision: int pass - def zero_remainder(self, x): - number = x + def generate_id(self): + return uuid4() + + def simulate_order(self, pair, qty, side): + """ + Price is determined by market + to help trigger the order immediately + """ + price = float(self.matching_engine(pair, True, qty)) + order = { + "symbol": pair, + "orderId": self.generate_id().int, + "orderListId": -1, + "clientOrderId": self.generate_id().hex, + "transactTime": time() * 1000, + "price": price, + "origQty": qty, + "executedQty": qty, + "cummulativeQuoteQty": qty, + "status": "FILLED", + "timeInForce": "GTC", + "type": "LIMIT", + "side": side, + "fills": [], + } + return order - while True: - if number % x == 0: - return number - else: - number += x + def simulate_response_order(self, pair, qty, side): + price = float(self.matching_engine(pair, True, qty)) + response_order = { + "symbol": pair, + "orderId": self.generate_id().int, + "orderListId": -1, + "clientOrderId": self.generate_id().hex, + "transactTime": time() * 1000, + "price": price, + "origQty": qty, + "executedQty": qty, + "cummulativeQuoteQty": qty, + "status": "FILLED", + "timeInForce": "GTC", + "type": "LIMIT", + "side": side, + "fills": [], + } + return response_order + + def simulate_margin_order(self, pair, side: OrderSide): + """ + Quantity doesn't matter, as it is not a real order that needs + to be process by the exchange + + Args: + - symbol and pair are used interchangably + - side: buy or sell + """ + qty = 1 + price = float(self.matching_engine(pair, True, qty)) + order = { + "symbol": pair, + "orderId": self.generate_id().int, + "orderListId": -1, + "clientOrderId": self.generate_id().hex, + "transactTime": time() * 1000, + "price": price, + "origQty": qty, + "executedQty": qty, + "cummulativeQuoteQty": qty, + "status": "FILLED", + "timeInForce": "GTC", + "type": "LIMIT", + "side": side, + "marginBuyBorrowAmount": 5, + "marginBuyBorrowAsset": "BTC", + "isIsolated": "true", + "fills": [], + } + return order def sell_order(self, symbol, qty, price=None): """ @@ -82,7 +156,7 @@ def buy_order(self, symbol, qty, price=None): # If price is not provided by matching engine, # create iceberg orders if not book_price: - payload["iceberg_qty"] = self.zero_remainder(qty) + payload["iceberg_qty"] = zero_remainder(qty) payload["price"] = supress_notation(book_price, self.price_precision) else: diff --git a/api/paper_trading/routes.py b/api/paper_trading/routes.py index 66a948262..e520963fb 100644 --- a/api/paper_trading/routes.py +++ b/api/paper_trading/routes.py @@ -1,5 +1,17 @@ -from fastapi import APIRouter, HTTPException, Query -from bots.controllers import Bot +import json +from fastapi import APIRouter, Depends, Query +from fastapi.encoders import jsonable_encoder +from sqlmodel import Session +from database.models.paper_trading_table import PaperTradingTable +from database.paper_trading_crud import PaperTradingTableCrud +from database.utils import get_session +from deals.controllers import CreateDealController +from tools.exceptions import BinanceErrors, BinbotErrors +from tools.handle_error import ( + json_response, + json_response_error, + json_response_message, +) from bots.schemas import BotSchema from typing import List @@ -15,62 +27,101 @@ def get( start_date: float | None = None, end_date: float | None = None, no_cooldown: bool = True, + session: Session = Depends(get_session), ): - return Bot(collection_name="paper_trading").get( - status, start_date, end_date, no_cooldown - ) + try: + bot = PaperTradingTableCrud(session=session).get( + status, start_date, end_date, no_cooldown + ) + return json_response({"message": "Bots found!", "data": jsonable_encoder(bot)}) + + except BinbotErrors as error: + return json_response_error(error) @paper_trading_blueprint.get("/paper-trading/{id}", tags=["paper trading"]) -def get_one(id: str): - return Bot(collection_name="paper_trading").get_one(id) +def get_one( + id: str, + session: Session = Depends(get_session), +): + try: + bot = PaperTradingTableCrud(session=session).get_one(bot_id=id, symbol=None) + if not bot: + return json_response_error("Bot not found.") + else: + return json_response({"message": "Bot found", "data": bot}) + except ValueError as error: + return json_response_error(error) @paper_trading_blueprint.post("/paper-trading", tags=["paper trading"]) -def create(bot_item: BotSchema): - return Bot(collection_name="paper_trading").create(bot_item) +def create(bot_item: BotSchema, session: Session = Depends(get_session)): + try: + bot = PaperTradingTableCrud(session=session).create(bot_item) + return json_response({"message": "Bot created", "data": bot}) + except BinbotErrors as error: + return json_response_error(error) @paper_trading_blueprint.put("/paper-trading/{id}", tags=["paper trading"]) -def edit(id: str, data: BotSchema): - return Bot(collection_name="paper_trading").edit(id, data) +def edit(id: str, bot_item: BotSchema, session: Session = Depends(get_session)): + try: + bot = PaperTradingTableCrud(session=session).create(bot_item) + return json_response({"message": "Bot updated", "data": bot}) + except BinbotErrors as error: + return json_response_error(error) @paper_trading_blueprint.delete("/paper-trading", tags=["paper trading"]) -def delete(id: List[str] = Query(...)): +def delete(id: List[str] = Query(...), session: Session = Depends(get_session)): """ Receives a list of `id=a1b2c3&id=b2c3d4` """ - return Bot(collection_name="paper_trading").delete(id) + try: + PaperTradingTableCrud(session=session).delete(id) + except BinbotErrors as error: + return json_response_error(error) @paper_trading_blueprint.get("/paper-trading/activate/{id}", tags=["paper trading"]) -def activate(id: str): - bot_instance = Bot(collection_name="paper_trading") - bot = bot_instance.get_one(id) +def activate(id: str, session: Session = Depends(get_session)): + bot = PaperTradingTableCrud(session=session).get_one(bot_id=id) if not bot: - raise HTTPException( - status_code=404, detail="Could not activate a bot that doesn't exist" - ) + return json_response_error("Bot not found.") + + bot_instance = CreateDealController(bot, db_table=PaperTradingTable) - botSchema = BotSchema.model_validate(bot) - return Bot(collection_name="paper_trading").activate(botSchema) + try: + bot_instance.open_deal() + return json_response_message("Successfully activated bot!") + + except BinbotErrors as error: + bot_instance.controller.update_logs(bot_id=id, log_message=error.message) + return json_response_error(error.message) + except BinanceErrors as error: + bot_instance.controller.update_logs(bot_id=id, log_message=error.message) + return json_response_error(error.message) @paper_trading_blueprint.delete( "/paper-trading/deactivate/{id}", tags=["paper trading"] ) -def deactivate(id: str): +def deactivate(id: str, session: Session = Depends(get_session)): """ Deactivation means closing all deals and selling to GBP Otherwise losses will be incurred """ - bot_instance = Bot(collection_name="paper_trading") - bot = bot_instance.get_one(id) - if not bot: - raise HTTPException( - status_code=404, detail="Could not deactivate a bot that doesn't exist" + bot_model = PaperTradingTableCrud(session=session).get_one(bot_id=id) + if not bot_model: + return json_response_error("No active bot found. Can't deactivate") + + bot_instance = CreateDealController(bot_model, db_table=PaperTradingTable) + try: + bot_instance.close_all() + return json_response_message( + "Active orders closed, sold base asset, deactivated" ) - botSchema = BotSchema.model_validate(bot) - return Bot(collection_name="paper_trading").deactivate(botSchema) + except BinbotErrors as error: + bot_instance.controller.update_logs(bot_id=id, log_message=error.message) + return json_response_error(error.message) diff --git a/api/streaming/streaming_controller.py b/api/streaming/streaming_controller.py index 41329809d..fb1a2412f 100644 --- a/api/streaming/streaming_controller.py +++ b/api/streaming/streaming_controller.py @@ -1,62 +1,62 @@ import json import logging import typing - +from kafka import KafkaConsumer +from database.models.bot_table import BotTable +from database.models.paper_trading_table import PaperTradingTable +from database.paper_trading_crud import PaperTradingTableCrud +from database.bot_crud import BotTableCrud +from deals.controllers import CreateDealController from tools.round_numbers import round_numbers from streaming.models import SignalsConsumer -from bots.schemas import BotSchema from autotrade.controller import AutotradeSettingsController -from bots.controllers import Bot from tools.enum_definitions import Status, Strategy -from database.db import Database from deals.margin import MarginDeal from deals.spot import SpotLongDeal from tools.exceptions import BinanceErrors -class BaseStreaming(Database): - def get_current_bot(self, symbol): - current_bot = Bot(collection_name="bots").get_one( - symbol=symbol, status=Status.active - ) +class BaseStreaming: + def __init__(self) -> None: + self.bot_controller = BotTableCrud() + self.paper_trading_controller = PaperTradingTableCrud() + + def get_current_bot(self, symbol: str) -> BotTable: + current_bot = self.bot_controller.get_one(symbol=symbol, status=Status.active) return current_bot - def get_current_test_bot(self, symbol): - current_test_bot = Bot(collection_name="paper_trading").get_one( + def get_current_test_bot(self, symbol: str) -> PaperTradingTable: + current_test_bot = self.paper_trading_controller.get_one( symbol=symbol, status=Status.active ) - current_test_bot = current_test_bot return current_test_bot class StreamingController(BaseStreaming): - def __init__(self, consumer): + def __init__(self, consumer: KafkaConsumer) -> None: super().__init__() - self.streaming_db = self._db # Gets any signal to restart streaming self.consumer = consumer self.autotrade_controller = AutotradeSettingsController() self.load_data_on_start() - def load_data_on_start(self): + def load_data_on_start(self) -> None: """ New function to replace get_klines without websockets """ # Load real bot settings - bot_controller = Bot(collection_name="bots") - self.list_bots = bot_controller.get_active_pairs() + self.list_bots = self.bot_controller.get_active_pairs() # Load paper trading bot settings - paper_trading_controller_paper = Bot(collection_name="paper_trading") - self.list_paper_trading_bots = paper_trading_controller_paper.get_active_pairs() + self.list_paper_trading_bots = self.paper_trading_controller.get_active_pairs() return def execute_strategies( self, - current_bot, + current_bot: BotTable | PaperTradingTable, close_price: str, open_price: str, - db_collection_name, - ): + create_deal_controller: CreateDealController, + ) -> None: """ Processes the deal market websocket price updates @@ -69,44 +69,24 @@ def execute_strategies( except Exception: print(current_bot["orders"][0]["order_id"]) pass - try: - active_bot = BotSchema(**current_bot) - pass - except Exception as error: - logging.info(error) - return + + active_bot = BotTable.model_validate(current_bot) + # Margin short if active_bot.strategy == Strategy.margin_short: - margin_deal = MarginDeal(active_bot, db_collection_name) - try: - margin_deal.streaming_updates(close_price) - except BinanceErrors as error: - if error.code in (-2010, -1013): - margin_deal.update_deal_logs(error.message, active_bot) - except Exception as error: - logging.info(error) - margin_deal.update_deal_logs(error, active_bot) - pass + margin_deal = MarginDeal(active_bot, create_deal_controller.controller) + margin_deal.streaming_updates(close_price) else: # Long strategy starts if active_bot.strategy == Strategy.long: - spot_long_deal = SpotLongDeal(active_bot, db_collection_name) - try: - spot_long_deal.streaming_updates(close_price, open_price) - except BinanceErrors as error: - if error.code in (-2010, -1013): - spot_long_deal.update_deal_logs(error.message, active_bot) - active_bot.status = Status.error - active_bot = self.save_bot_streaming(active_bot) - except Exception as error: - logging.info(error) - spot_long_deal.update_deal_logs(error, active_bot) - pass - + spot_long_deal = SpotLongDeal( + active_bot, create_deal_controller.controller + ) + spot_long_deal.streaming_updates(close_price, open_price) pass - def process_klines(self, message): + def process_klines(self, message: str) -> None: """ Updates deals with klines websockets, when price and symbol match existent deal @@ -121,46 +101,59 @@ def process_klines(self, message): # temporary test that we get enough streaming update signals logging.info(f"Streaming update for {symbol}") - if current_bot: - self.execute_strategies( - current_bot, - close_price, - open_price, - "bots", - ) - if current_test_bot: - self.execute_strategies( - current_test_bot, - close_price, - open_price, - "paper_trading", - ) + try: + if current_bot: + create_deal_controller = CreateDealController( + bot=current_bot, controller=BotTableCrud + ) + self.execute_strategies( + current_bot, + close_price, + open_price, + create_deal_controller, + ) + if current_test_bot: + create_deal_controller = CreateDealController( + bot=current_bot, controller=BotTableCrud + ) + self.execute_strategies( + current_test_bot, + close_price, + open_price, + create_deal_controller, + ) + except BinanceErrors as error: + if error.code in (-2010, -1013): + bot = current_bot if current_bot else current_test_bot + create_deal_controller.controller.update_logs( + error.message, bot + ) + bot.status = Status.error + create_deal_controller.controller.save(bot) return class BbspreadsUpdater(BaseStreaming): - def __init__(self): - self.current_bot: BotSchema | None = None - self.current_test_bot: BotSchema | None = None + def __init__(self) -> None: + self.current_bot: BotTable | None = None + self.current_test_bot: PaperTradingTable | None = None - def load_current_bots(self, symbol): + def load_current_bots(self, symbol: str) -> None: current_bot_payload = self.get_current_bot(symbol) if current_bot_payload: - self.current_bot = BotSchema(**current_bot_payload) + self.current_bot = BotTable.model_validate(current_bot_payload) current_test_bot_payload = self.get_current_test_bot(symbol) if current_test_bot_payload: - self.current_test_bot = BotSchema(**current_test_bot_payload) - - def reactivate_bot(self, bot: BotSchema, collection_name="bots"): - bot_instance = Bot(collection_name=collection_name) - activated_bot = bot_instance.activate(bot) - return activated_bot + self.current_test_bot = BotTable(**current_test_bot_payload) def update_bots_parameters( - self, bot: BotSchema, bb_spreads, collection_name="bots" - ): + self, + bot: BotTable, + bb_spreads: dict, + create_deal_controller: CreateDealController, + ) -> None: # multiplied by 1000 to get to the same scale stop_loss top_spread = round_numbers( ( @@ -207,7 +200,7 @@ def update_bots_parameters( # too much risk, reduce stop loss bot.trailling_deviation = bottom_spread # reactivate includes saving - self.reactivate_bot(bot, collection_name=collection_name) + create_deal_controller.open_deal(bot) # No need to continue # Bots can only be either long or short @@ -222,7 +215,7 @@ def update_bots_parameters( if bot.trailling_deviation > bottom_spread: bot.trailling_deviation = top_spread # reactivate includes saving - self.reactivate_bot(bot, collection_name=collection_name) + create_deal_controller.open_deal(bot) # To find a better interface for bb_xx once mature @typing.no_type_check @@ -233,7 +226,7 @@ def update_close_conditions(self, message): dynamic movements in the market """ data = json.loads(message) - signalsData = SignalsConsumer(**data) + signalsData = SignalsConsumer.model_validate(data) # Check if it matches any active bots self.load_current_bots(signalsData.symbol) @@ -249,8 +242,20 @@ def update_close_conditions(self, message): and bb_spreads["bb_mid"] ): if self.current_bot: - self.update_bots_parameters(self.current_bot, bb_spreads) + create_deal_controller = CreateDealController( + bot=self.current_bot, controller=BotTableCrud + ) + self.update_bots_parameters( + self.current_bot, + bb_spreads, + create_deal_controller=create_deal_controller, + ) if self.current_test_bot: + create_deal_controller = CreateDealController( + bot=self.current_test_bot, controller=PaperTradingTableCrud + ) self.update_bots_parameters( - self.current_test_bot, bb_spreads, collection_name="paper_trading" + self.current_test_bot, + bb_spreads, + create_deal_controller=create_deal_controller, ) diff --git a/api/tools/handle_error.py b/api/tools/handle_error.py index 8de3a8f17..77791ee10 100644 --- a/api/tools/handle_error.py +++ b/api/tools/handle_error.py @@ -62,7 +62,7 @@ def handle_binance_errors(response: Response) -> dict: sleep(120) if response.status_code == 418 or response.status_code == 429: - print("Request weight limit hit, ban will come soon, waiting 1 hour") + logging.warning("Request weight limit hit, ban will come soon, waiting 1 hour") sleep(3600) # Cloudfront 403 error diff --git a/api/tools/round_numbers.py b/api/tools/round_numbers.py index 6fed0b93d..d1fc2c49f 100644 --- a/api/tools/round_numbers.py +++ b/api/tools/round_numbers.py @@ -83,3 +83,13 @@ def format_ts(time: datetime.datetime) -> str: to human-readable date """ return time.strftime("%Y-%m-%d %H:%M:%S.%f") + + +def zero_remainder(x): + number = x + + while True: + if number % x == 0: + return number + else: + number += x diff --git a/api/user/models/user.py b/api/user/models/user.py index 512a4f0b3..212b92294 100644 --- a/api/user/models/user.py +++ b/api/user/models/user.py @@ -1,5 +1,5 @@ from typing import Optional -from pydantic import BaseModel, EmailStr, field_validator +from pydantic import BaseModel, EmailStr, SecretStr, field_validator from sqlmodel import Field from tools.enum_definitions import UserRoles from tools.handle_error import StandardResponse @@ -22,7 +22,7 @@ class CreateUser(BaseModel): is_active: bool = True role: UserRoles = Field(default=UserRoles.admin) full_name: Optional[str] = Field(default="") - password: str = Field(min_length=8, max_length=40) + password: SecretStr = Field(min_length=8, max_length=40) # Email is the main identifier username: Optional[str] = Field(default="") bio: Optional[str] = Field(default="")