From 756c481219cefda06ab4e0a0de155196074071e1 Mon Sep 17 00:00:00 2001 From: Carlos Wu Fei Date: Sun, 8 Dec 2024 21:56:02 +0000 Subject: [PATCH] Simplify database operations By creating simpler and atomic db operations as a separate layer, we reduce complexity for testing (duplicated code), decouple dependencies of database and services and also reduce circular imports. --- api/bots/routes.py | 43 ++++++----- api/bots/schemas.py | 14 +++- api/charts/controllers.py | 11 ++- .../bot_crud.py} | 71 ++++++++--------- api/database/paper_trading_crud.py | 77 +++++++++++++++++++ api/deals/base.py | 42 +++++----- api/deals/controllers.py | 31 ++++++-- api/orders/controller.py | 4 +- 8 files changed, 193 insertions(+), 100 deletions(-) rename api/{bots/bot_table_controller.py => database/bot_crud.py} (77%) create mode 100644 api/database/paper_trading_crud.py diff --git a/api/bots/routes.py b/api/bots/routes.py index 0510faf45..b27bfd2d2 100644 --- a/api/bots/routes.py +++ b/api/bots/routes.py @@ -1,4 +1,10 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends, HTTPException +from sqlmodel import Session +from api.database.bot_crud import BotTableCrud +from api.deals.controllers import CreateDealController +from bots.bot_table_controller import BotTableController +from database.models.bot_table import BotTable +from database.utils import get_session from tools.handle_error import json_response, json_response_error, json_response_message from bots.controllers import Bot from bots.schemas import BotSchema, BotListResponse, ErrorsRequestBody @@ -69,7 +75,7 @@ def delete(id: List[str]): @bot_blueprint.get("/bot/activate/{id}", tags=["bots"]) -async def activate_by_id(id: str): +async def activate_by_id(id: str, session: Session = Depends(get_session)): """ Activate bot @@ -77,21 +83,21 @@ async def activate_by_id(id: str): - If changes were made, it will override DB data - Because botId is received from endpoint, it will be a str not a PyObjectId """ - bot_instance = Bot(collection_name="bots") - bot = bot_instance.get_one(id) - if bot: - try: - bot_instance.activate(bot) - return json_response_message("Successfully activated bot!") - except BinbotErrors as error: - bot_instance.post_errors_by_id(id, error.message) - return json_response_error(error.message) - except BinanceErrors as error: - bot_instance.post_errors_by_id(id, error.message) - return json_response_error(error.message) + bot = BotTableCrud(session=session).get_one(bot_id=id) + if not bot: + return json_response_message("Successfully activated bot!") - else: - return json_response_error("Bot not found.") + bot_instance = CreateDealController(bot, db_table=BotTable) + + try: + bot_instance.activate(bot) + return json_response_message("Successfully activated bot!") + except BinbotErrors as error: + bot_instance.update_logs(bot_id=id, log_message=error.message) + return json_response_error(error.message) + except BinanceErrors as error: + bot_instance.update_logs(bot_id=id, log_message=error.message) + return json_response_error(error.message) @bot_blueprint.delete("/bot/deactivate/{id}", tags=["bots"]) @@ -115,11 +121,6 @@ def deactivation(id: str): return json_response_error("Error deactivating bot.") -@bot_blueprint.put("/bot/archive/{id}", tags=["bots"]) -def archive(id: str): - return Bot(collection_name="bots").put_archive(id) - - @bot_blueprint.post("/bot/errors/{bot_id}", tags=["bots"]) def bot_errors(bot_id: str, bot_errors: ErrorsRequestBody): """ diff --git a/api/bots/schemas.py b/api/bots/schemas.py index 2e0dd1450..c264c0e7a 100644 --- a/api/bots/schemas.py +++ b/api/bots/schemas.py @@ -22,7 +22,9 @@ class BotSchema(BaseModel): fiat: str = "USDC" balance_to_use: str = "USDC" base_order_size: float | int = 15 # Min Binance 0.0001 BNB - candlestick_interval: BinanceKlineIntervals = Field(default=BinanceKlineIntervals.fifteen_minutes) + candlestick_interval: BinanceKlineIntervals = Field( + default=BinanceKlineIntervals.fifteen_minutes + ) close_condition: CloseConditions = Field(default=CloseConditions.dynamic_trailling) # cooldown period in minutes before opening next bot with same pair cooldown: int = 0 @@ -91,7 +93,9 @@ def check_names_not_empty(cls, v): assert v != "", "Empty pair field." return v - @field_validator("balance_size_to_use", "base_order_size", "base_order_size", mode="before") + @field_validator( + "balance_size_to_use", "base_order_size", "base_order_size", mode="before" + ) @classmethod def countables(cls, v): if isinstance(v, float): @@ -104,7 +108,11 @@ def countables(cls, v): raise ValueError(f"{v} must be a number (float, int or string)") @field_validator( - "stop_loss", "take_profit", "trailling_deviation", "trailling_profit", mode="before" + "stop_loss", + "take_profit", + "trailling_deviation", + "trailling_profit", + mode="before", ) @classmethod def check_percentage(cls, v): diff --git a/api/charts/controllers.py b/api/charts/controllers.py index f3bc8aa30..20eb84f8e 100644 --- a/api/charts/controllers.py +++ b/api/charts/controllers.py @@ -248,5 +248,14 @@ def top_gainers(self): fiat = self.autotrade_db.get_fiat() ticket_data = self.ticker_24() - fiat_market_data = sorted((item for item in ticket_data if item["symbol"].endswith(fiat) and float(item["priceChangePercent"]) > 0), key=lambda x: x["priceChangePercent"], reverse=True) + fiat_market_data = sorted( + ( + item + for item in ticket_data + if item["symbol"].endswith(fiat) + and float(item["priceChangePercent"]) > 0 + ), + key=lambda x: x["priceChangePercent"], + reverse=True, + ) return fiat_market_data[:10] diff --git a/api/bots/bot_table_controller.py b/api/database/bot_crud.py similarity index 77% rename from api/bots/bot_table_controller.py rename to api/database/bot_crud.py index 8828cf10f..b68ad1357 100644 --- a/api/bots/bot_table_controller.py +++ b/api/database/bot_crud.py @@ -3,21 +3,24 @@ from fastapi import Query from sqlmodel import Session, asc, desc, or_, select, case from time import time -from base_producer import BaseProducer +from deals.controllers import CreateDealController from bots.schemas import BotSchema from database.models.bot_table import BotTable from database.models.deal_table import DealTable from database.utils import independent_session -from deals.controllers import CreateDealController from deals.models import DealModel -from tools.enum_definitions import BinbotEnums, Status -from psycopg.types.json import Json, set_json_loads +from tools.enum_definitions import BinbotEnums, Status, Strategy +from psycopg.types.json import Json -class BotTableController: +class BotTableCrud: """ CRUD and database operations for the SQL API DB bot_table table. + + Use for lower level APIs that require a session + e.g. + client-side -> receive json -> bots.routes -> BotTableCrud """ def __init__( @@ -29,11 +32,10 @@ def __init__( if session is None: session = independent_session() self.session = session - self.base_producer = BaseProducer() - self.producer = self.base_producer.start_producer() - self.deal: CreateDealController | None = None - def update_logs(self, log_message: str, bot: BotSchema = None, bot_id: str | None = None): + def update_logs( + self, log_message: str, bot: BotSchema = None, bot_id: str | None = None + ): """ Update logs for a bot @@ -74,8 +76,10 @@ def get( """ Get all bots in the db except archived Args: - - archive=false - - filter_by: string - last-week, last-month, all + - status: Status enum + - start_date and end_date are timestamps in milliseconds + - no_cooldown: bool - filter out bots that are in cooldown + - limit and offset for pagination """ statement = select(BotTable) @@ -143,6 +147,9 @@ def get_one(self, bot_id: str | None = None, symbol: str | None = None): def create(self, data: BotSchema): """ Create a new bot + + It's crucial to reset fields, so bot can trigger base orders + and start trailling. """ bot = BotTable.model_validate(data) # Ensure values are reset @@ -152,26 +159,27 @@ def create(self, data: BotSchema): bot.updated_at = time() * 1000 bot.status = Status.inactive bot.deal = DealModel() + + # db operations self.session.add(bot) self.session.commit() self.session.close() - self.base_producer.update_required(self.producer, "CREATE_BOT") return bot - def edit(self, bot_id: str, data: BotSchema): + def edit(self, id: str, data: BotSchema): """ Edit a bot """ - bot = self.session.get(BotTable, bot_id) + bot = self.session.get(BotTable, id) if not bot: return bot - dumped_bot = bot.model_dump(exclude_unset=True) + # double check orders and deal are not overwritten + dumped_bot = data.model_dump(exclude_unset=True) bot.sqlmodel_update(dumped_bot) self.session.add(bot) self.session.commit() self.session.close() - self.base_producer.update_required(self.producer, "UPDATE_BOT") return bot def delete(self, bot_ids: List[str] = Query(...)): @@ -185,38 +193,19 @@ def delete(self, bot_ids: List[str] = Query(...)): bots = self.session.exec(statement).all() self.session.commit() self.session.close() - self.base_producer.update_required(self.producer, "DELETE_BOT") return bots - def activate(self, bot_id: str): - """ - Activate a bot - """ - bot = self.session.get(BotTable, bot_id) - if not bot: - return bot - - bot.status = Status.active - self.session.add(bot) - self.session.commit() - self.session.close() - self.base_producer.update_required(self.producer, "ACTIVATE_BOT") - return bot - - def deactivate(self, bot_id: str): + def update_status(self, bot: BotTable, status: Status) -> BotTable: """ - Deactivate a bot (panic sell) + Activate a bot by opening a deal """ - bot = self.session.get(BotTable, bot_id) - if not bot: - return bot - - bot.status = Status.completed - + bot.status = status + # db operations self.session.add(bot) self.session.commit() self.session.close() - self.base_producer.update_required(self.producer, "DEACTIVATE_BOT") + # do this after db operations in case there is rollback + # avoids sending unnecessary signals return bot def get_active_pairs(self): diff --git a/api/database/paper_trading_crud.py b/api/database/paper_trading_crud.py new file mode 100644 index 000000000..8fdf328f1 --- /dev/null +++ b/api/database/paper_trading_crud.py @@ -0,0 +1,77 @@ +from time import time + +from sqlmodel import Session +from database.models.paper_trading_table import PaperTradingTable +from database.utils import independent_session +from tools.enum_definitions import Status + + +class PaperTradingTableCrud: + def __init__(self, session: Session | None = None): + if session is None: + session = independent_session() + self.session = session + pass + + def create(self, paper_trading: PaperTradingTable) -> PaperTradingTable: + """ + Create a new paper trading account + """ + paper_trading.created_at = time() * 1000 + paper_trading.updated_at = time() * 1000 + + # db operations + self.session.add(paper_trading) + self.session.commit() + self.session.close() + return paper_trading + + def edit(self, paper_trading: PaperTradingTable) -> PaperTradingTable: + """ + Edit a paper trading account + """ + self.session.add(paper_trading) + self.session.commit() + self.session.close() + return paper_trading + + def delete(self, id: str) -> bool: + """ + Delete a paper trading account by id + """ + paper_trading = self.session.get(PaperTradingTable, id) + if not paper_trading: + return False + + self.session.delete(paper_trading) + self.session.commit() + self.session.close() + return True + + def get(self, id: str) -> PaperTradingTable: + """ + Get a paper trading account by id + """ + paper_trading = self.session.get(PaperTradingTable, id) + self.session.close() + return paper_trading + + def activate(self, paper_trading: PaperTradingTable) -> PaperTradingTable: + """ + Activate a paper trading account + """ + paper_trading.status = Status.active + self.session.add(paper_trading) + self.session.commit() + self.session.close() + return paper_trading + + def deactivate(self, paper_trading: PaperTradingTable) -> PaperTradingTable: + """ + Deactivate a paper trading account + """ + paper_trading.status = Status.inactive + self.session.add(paper_trading) + self.session.commit() + self.session.close() + return paper_trading diff --git a/api/deals/base.py b/api/deals/base.py index bd4f015ce..b98621a4a 100644 --- a/api/deals/base.py +++ b/api/deals/base.py @@ -1,13 +1,15 @@ from typing import Tuple import uuid from time import time -from pymongo import ReturnDocument from datetime import datetime + +from database.bot_crud import BotTableCrud +from database.paper_trading_crud import PaperTradingTableCrud +from database.models.paper_trading_table import PaperTradingTable +from database.models.bot_table import BotTable from deals.models import BinanceOrderModel, DealModel from orders.controller import OrderController -from bots.schemas import BotSchema from tools.round_numbers import round_numbers, supress_notation, round_numbers_ceiling -from tools.handle_error import encode_json from tools.exceptions import ( BinanceErrors, DealCreationError, @@ -15,6 +17,7 @@ MarginLoanNotFound, ) from tools.enum_definitions import DealType, Status, Strategy +from base_producer import BaseProducer # To be removed one day en commission endpoint found that provides this value @@ -31,15 +34,14 @@ class BaseDeal(OrderController): self.symbol is always the same. """ - def __init__(self, bot, db_collection_name): - if not isinstance(bot, BotSchema): - self.active_bot = BotSchema(**bot) - else: - self.active_bot = bot - self.db_collection = self._db[db_collection_name] - self.market_domination_reversal = None + def __init__(self, bot: BotTable, controller: PaperTradingTableCrud | BotTableCrud): + self.active_bot = bot + self.controller: PaperTradingTableCrud | BotTable = controller + self.market_domination_reversal: bool | None = None self.price_precision = self.calculate_price_precision(bot.pair) self.qty_precision = self.calculate_qty_precision(bot.pair) + self.base_producer = BaseProducer() + self.producer = self.base_producer.start_producer() if self.active_bot.strategy == Strategy.margin_short: self.isolated_balance = self.get_isolated_balance(self.active_bot.pair) @@ -226,7 +228,7 @@ def base_order(self): if float(self.active_bot.stop_loss) > 0: stop_loss_price = price - (price * (float(self.active_bot.stop_loss) / 100)) - if self.db_collection.name == "paper_trading": + if self.controller == PaperTradingTable: res = self.simulate_order( self.active_bot.pair, qty, @@ -265,18 +267,10 @@ def base_order(self): ) # Activate bot - self.active_bot.status = Status.active - - bot = encode_json(self.active_bot) - if "_id" in bot: - bot.pop("_id") # _id is what causes conflict not id - - document = self.db_collection.find_one_and_update( - {"id": self.active_bot.id}, - {"$set": bot}, - return_document=ReturnDocument.AFTER, - ) - + document = self.controller.activate(self.active_bot) + # do this after db operations in case there is rollback + # avoids sending unnecessary signals + self.base_producer.update_required(self.producer, "ACTIVATE_BOT") return document def margin_liquidation(self, pair: str): @@ -285,7 +279,7 @@ def margin_liquidation(self, pair: str): Args: - pair: a.k.a symbol, quote asset + base asset - - qty_precision: to round numbers for Binance API. Passed optionally to + - qty_precision: to round numbers for Binance Passed optionally to reduce number of requests to avoid rate limit. """ self.isolated_balance = self.get_isolated_balance(pair) diff --git a/api/deals/controllers.py b/api/deals/controllers.py index d41267bed..f48c9ee60 100644 --- a/api/deals/controllers.py +++ b/api/deals/controllers.py @@ -1,3 +1,7 @@ +from database.bot_crud import BotTableCrud +from database.paper_trading_crud import PaperTradingTableCrud +from database.models.paper_trading_table import PaperTradingTable +from database.models.bot_table import BotTable from orders.controller import OrderController from bots.schemas import BotSchema from deals.base import BaseDeal @@ -23,13 +27,24 @@ class CreateDealController(BaseDeal): 3. Update deals (deal update controller) - db_collection = ["bots", "paper_trading"]. - paper_trading uses simulated orders and bot uses real binance orders + paper_trading uses simulated orders and bot uses real binance orders. + PaperTradingTable is implemented, PaperTradingController with the db operations is not. + - bot: BotSchema (at some point to refactor into BotTable as they are both pydantic models) """ - def __init__(self, bot: BotSchema, db_collection="paper_trading"): - # Inherit from parent class - super().__init__(bot, db_collection) + def __init__( + self, + bot: BotSchema | BotTable, + db_table: PaperTradingTable | BotTable = BotTable, + ): + if db_table == PaperTradingTable: + db_controller = PaperTradingTableCrud + else: + db_controller = BotTableCrud + + super().__init__(bot, db_controller) self.active_bot = bot + self.db_table = db_table def compute_qty(self, pair): """ @@ -55,7 +70,7 @@ def take_profit_order(self) -> BotSchema: buy_total_qty = self.active_bot.deal.buy_total_qty price = (1 + (float(self.active_bot.take_profit) / 100)) * float(deal_buy_price) - if self.db_collection.name == "paper_trading": + if self.db_table == "paper_trading": qty = self.active_bot.deal.buy_total_qty else: qty = self.compute_qty(self.active_bot.pair) @@ -63,7 +78,7 @@ def take_profit_order(self) -> BotSchema: qty = round_numbers(buy_total_qty, self.qty_precision) price = round_numbers(price, self.price_precision) - if self.db_collection.name == "paper_trading": + if self.db_table == "paper_trading": res = self.simulate_order(self.active_bot.pair, qty, "SELL") if price: res = self.simulate_order( @@ -253,7 +268,7 @@ def open_deal(self) -> None: if not base_order_deal: if self.active_bot.strategy == Strategy.margin_short: self.active_bot = MarginDeal( - bot=self.active_bot, db_collection_name=self.db_collection.name + bot=self.active_bot, db_collection_name=self.db_table ).margin_short_base_order() else: bot = self.base_order() @@ -269,7 +284,7 @@ def open_deal(self) -> None: if float(self.active_bot.stop_loss) > 0: if self.active_bot.strategy == Strategy.margin_short: self.active_bot = MarginDeal( - bot=self.active_bot, db_collection_name=self.db_collection.name + bot=self.active_bot, db_collection_name=self.db_table ).set_margin_short_stop_loss() else: buy_price = float(self.active_bot.deal.buy_price) diff --git a/api/orders/controller.py b/api/orders/controller.py index 048b5ada5..c1849b7aa 100644 --- a/api/orders/controller.py +++ b/api/orders/controller.py @@ -1,12 +1,12 @@ from account.account import Account +from api.database.bot_crud import BotTableCrud from tools.exceptions import DeleteOrderError -from database.db import Database from tools.enum_definitions import OrderType, TimeInForce, OrderSide from tools.handle_error import json_response, json_response_message from tools.round_numbers import supress_notation -class OrderController(Database, Account): +class OrderController(BotTableCrud, Account): """ Always GTC and limit orders limit/market orders will be decided by matching_engine