Refactor database layer (#558)

* Fix asset test and setup abstraction for database (#557)

Various fixes for pydantic upgrade

* Move database instance to controllers

* Refactor Research endpoints

api/account/assets.py

@@ -15,7 +15,6 @@
 
 class Assets(BaseDeal):
     def __init__(self):
-        self.db = Database()
         self.usd_balance = 0
         self.exception_list = []
 
@@ -444,45 +443,7 @@ def get_market_domination(self, size=7):
         Returns:
             dict: A dictionary containing the market domination data, including gainers and losers counts, percentages, and dates.
         """
-        try:
-            data = self.db.query_market_domination(size=size)
-            market_domination_series = MarketDominationSeries()
-
-            for item in data:
-                gainers_percent = 0
-                losers_percent = 0
-                gainers_count = 0
-                losers_count = 0
-                total_volume = 0
-                if "data" in item:
-                    for crypto in item["data"]:
-                        if float(crypto['priceChangePercent']) > 0:
-                            gainers_percent += float(crypto['volume'])
-                            gainers_count += 1
-
-                        if float(crypto['priceChangePercent']) < 0:
-                            losers_percent += abs(float(crypto['volume']))
-                            losers_count += 1
-
-                        if float(crypto['volume']) > 0:
-                            total_volume += float(crypto['volume']) * float(crypto['price'])
-
-                market_domination_series.dates.append(item["time"])
-                market_domination_series.gainers_percent.append(gainers_percent)
-                market_domination_series.losers_percent.append(losers_percent)
-                market_domination_series.gainers_count.append(gainers_count)
-                market_domination_series.losers_count.append(losers_count)
-                market_domination_series.total_volume.append(total_volume)
-
-            market_domination_series.dates = market_domination_series.dates[-size:]
-            market_domination_series.gainers_percent = market_domination_series.gainers_percent[-size:]
-            market_domination_series.losers_percent = market_domination_series.losers_percent[-size:]
-            market_domination_series.gainers_count = market_domination_series.gainers_count[-size:]
-            market_domination_series.losers_count = market_domination_series.losers_count[-size:]
-            market_domination_series.total_volume = market_domination_series.total_volume[-size:]
-
-            data = market_domination_series.model_dump(mode="json")
-
-            return json_response({ "data": data, "message": "Successfully retrieved market domination data.", "error": 0 })
-        except Exception as error:
-            return json_response_error(f"Failed to retrieve market domination data: {error}")
+        query = {"$query": {}, "$orderby": {"_id": -1}}
+        result = self._db.market_domination.find(query).limit(size)
+        return list(result)
+

api/account/routes.py

@@ -2,7 +2,15 @@
 from datetime import datetime, timedelta
 from account.account import Account
 from account.assets import Assets
-from account.schemas import BalanceResponse, GainersLosersResponse, BalanceSeriesResponse, GetMarketDominationResponse, MarketDominationResponse
+from account.schemas import (
+    BalanceResponse,
+    GainersLosersResponse,
+    BalanceSeriesResponse,
+    GetMarketDominationResponse,
+    MarketDominationResponse,
+    MarketDominationSeries,
+)
+from tools.handle_error import json_response, json_response_error
 
 account_blueprint = APIRouter()
 
@@ -70,32 +78,112 @@ def store_balance():
     return Assets().store_balance()
 
 
-@account_blueprint.get("/gainers-losers", response_model=GainersLosersResponse, tags=["assets"])
+@account_blueprint.get(
+    "/gainers-losers", response_model=GainersLosersResponse, tags=["assets"]
+)
 async def retrieve_gainers_losers():
     return await Assets().retrieve_gainers_losers()
 
-@account_blueprint.get("/balance-series", response_model=BalanceSeriesResponse, tags=["assets"])
+
+@account_blueprint.get(
+    "/balance-series", response_model=BalanceSeriesResponse, tags=["assets"]
+)
 async def get_balance_series():
     today = datetime.now()
     month_ago = today - timedelta(30)
-    return await Assets().get_balance_series(start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today))
+    return await Assets().get_balance_series(
+        start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today)
+    )
+
 
 @account_blueprint.get("/clean", response_model=BalanceSeriesResponse, tags=["assets"])
 def clean_balance():
     return Assets().clean_balance_assets()
 
-@account_blueprint.get("/disable-isolated", response_model=BalanceSeriesResponse, tags=["assets"])
+
+@account_blueprint.get(
+    "/disable-isolated", response_model=BalanceSeriesResponse, tags=["assets"]
+)
 def disable_isolated():
     return Assets().disable_isolated_accounts()
 
+
 @account_blueprint.get("/one-click-liquidation/{asset}", tags=["assets"])
 def one_click_liquidation(asset):
     return Assets().one_click_liquidation(asset)
 
-@account_blueprint.get("/market-domination", tags=["assets"], response_model=MarketDominationResponse)
-def market_domination(size: int=7):
-    return Assets().get_market_domination(size)
 
-@account_blueprint.get("/store-market-domination", tags=["assets"], response_model=GetMarketDominationResponse)
+@account_blueprint.get(
+    "/market-domination", tags=["assets"], response_model=MarketDominationResponse
+)
+def market_domination(size: int = 7):
+
+    data = Assets().get_market_domination(size)
+    market_domination_series = MarketDominationSeries()
+
+    try:
+        for item in data:
+            gainers_percent = 0
+            losers_percent = 0
+            gainers_count = 0
+            losers_count = 0
+            total_volume = 0
+            if "data" in item:
+                for crypto in item["data"]:
+                    if float(crypto['priceChangePercent']) > 0:
+                        gainers_percent += float(crypto['volume'])
+                        gainers_count += 1
+
+                    if float(crypto['priceChangePercent']) < 0:
+                        losers_percent += abs(float(crypto['volume']))
+                        losers_count += 1
+
+                    if float(crypto['volume']) > 0:
+                        total_volume += float(crypto['volume']) * float(crypto['price'])
+
+            market_domination_series.dates.append(item["time"])
+            market_domination_series.gainers_percent.append(gainers_percent)
+            market_domination_series.losers_percent.append(losers_percent)
+            market_domination_series.gainers_count.append(gainers_count)
+            market_domination_series.losers_count.append(losers_count)
+            market_domination_series.total_volume.append(total_volume)
+
+        market_domination_series.dates = market_domination_series.dates[-size:]
+        market_domination_series.gainers_percent = (
+            market_domination_series.gainers_percent[-size:]
+        )
+        market_domination_series.losers_percent = (
+            market_domination_series.losers_percent[-size:]
+        )
+        market_domination_series.gainers_count = market_domination_series.gainers_count[
+            -size:
+        ]
+        market_domination_series.losers_count = market_domination_series.losers_count[
+            -size:
+        ]
+        market_domination_series.total_volume = market_domination_series.total_volume[
+            -size:
+        ]
+
+        data = market_domination_series.model_dump(mode="json")
+
+        return json_response(
+            {
+                "data": data,
+                "message": "Successfully retrieved market domination data.",
+                "error": 0,
+            }
+        )
+    except Exception as error:
+        return json_response_error(
+            f"Failed to retrieve market domination data: {error}"
+        )
+
+
+@account_blueprint.get(
+    "/store-market-domination",
+    tags=["assets"],
+    response_model=GetMarketDominationResponse,
+)
 def store_market_domination():
     return Assets().store_market_domination()

api/db.py

@@ -16,28 +16,20 @@ def setup_db():
 
 
 class Database:
-    def __init__(self):
-        self._db = setup_db()
-
-    def get_autotrade_settings(self):
-        return self._db.research_controller.find_one({"_id": "settings"})
-
-    def get_test_autotrade_settings(self):
-        return self._db.research_controller.find_one({"_id": "test_autotrade_settings"})
-
-    def get_active_bots(self):
-        bots = list(self._db.bots.distinct("pair", {"status": "active"}))
-        return bots
-
-    def get_active_paper_trading_bots(self):
-        bots = list(self._db.paper_trading.distinct("pair", {"status": "active"}))
-        return bots
-
-    def query_market_domination(self, size: int = 7):
-        """
-        Get market domination data and order by DESC _id,
-        which means earliest data will be at the end of the list.
-        """
-        query = {"$query": {}, "$orderby": {"_id": -1}}
-        result = self._db.market_domination.find(query).limit(size)
-        return list(result)
+    _db = setup_db()
+
+
+    # def get_autotrade_settings(self):
+    #     return self._db.research_controller.find_one({"_id": "settings"})
+
+    # def get_test_autotrade_settings(self):
+    #     return self._db.research_controller.find_one({"_id": "test_autotrade_settings"})
+
+    # def get_active_bots(self):
+    #     bots = list(self._db.bots.distinct("pair", {"status": "active"}))
+    #     return bots
+
+    # def get_active_paper_trading_bots(self):
+    #     bots = list(self._db.paper_trading.distinct("pair", {"status": "active"}))
+    #     return bots
+

api/deals/base.py

@@ -4,7 +4,7 @@
 import logging
 
 from time import time
-from db import setup_db
+from db import Database, setup_db
 from orders.controller import OrderController
 from bots.schemas import BotSchema
 from pymongo import ReturnDocument
@@ -16,7 +16,7 @@
 from tools.enum_definitions import Status, Strategy
 from bson.objectid import ObjectId
 from deals.schema import DealSchema, OrderSchema
-from datetime import datetime, timedelta
+from datetime import datetime
 
 # To be removed one day when commission endpoint found that provides this value
 ESTIMATED_COMMISSIONS_RATE = 0.0075
@@ -27,11 +27,10 @@ class BaseDeal(OrderController):
     """
 
     def __init__(self, bot, db_collection_name):
-        self.active_bot = BotSchema.parse_obj(bot)
+        self.active_bot = BotSchema(**bot)
         self.symbol = self.active_bot.pair
         super().__init__(symbol=self.active_bot.pair)
-        self.db = setup_db()
-        self.db_collection = self.db[db_collection_name]
+        self.db_collection = self._db[db_collection_name]
         self.market_domination_reversal = None
         if self.active_bot.strategy == Strategy.margin_short:
             self.isolated_balance: float = self.get_isolated_balance(self.symbol)
@@ -174,22 +173,6 @@ def close_open_orders(self, symbol):
                 return True
         return False
 
-    def update_required(self):
-        """
-        Terminate streaming and restart list of bots required
-
-        This will queue up a timer to restart streaming_controller when timer is reached
-        This timer is added, so that update_required petitions can be accumulated and
-        avoid successively restarting streaming_controller, which consumes a lot of memory
-
-        This means that everytime there is an update in the list of active bots,
-        it will reset the timer
-        """
-        self.db.research_controller.update_one(
-            {"_id": "settings"}, {"$set": {"update_required": time()}}
-        )
-        return
-
     def save_bot_streaming(self):
         """
         MongoDB query to save bot using Pydantic
@@ -234,10 +217,7 @@ def create_new_bot_streaming(self):
         bot = encode_json(self.active_bot)
         self.db_collection.insert_one(bot)
         new_bot = self.db_collection.find_one({"id": bot["id"]})
-        bot_class = BotSchema.parse_obj(new_bot)
-
-        # notify the system to update streaming as usual bot actions
-        self.db.research_controller.update_one({"_id": "settings"}, {"$set": {"update_required": time()}})
+        bot_class = BotSchema(**new_bot)
 
         return bot_class
 
@@ -318,7 +298,7 @@ def base_order(self):
         return document
 
     def dynamic_take_profit(self, current_bot, close_price):
-        self.active_bot = BotSchema.parse_obj(current_bot)
+        self.active_bot = BotSchema(**current_bot)
 
         params = {
             "symbol": self.active_bot.pair,

api/orders/controller.py

@@ -1,6 +1,7 @@
 import logging
 
 from account.account import Account
+from db import Database
 from tools.enum_definitions import OrderType, TimeInForce, OrderSide
 from tools.handle_error import json_response, json_response_error, json_response_message
 from tools.round_numbers import supress_notation
@@ -10,12 +11,15 @@
 poll_percentage = 0
 
 
-class OrderController(Account):
+class OrderController(Database, Account):
+    """
+    Always GTC and limit orders
+    limit/market orders will be decided by matching_engine
+    PRICE_FILTER decimals
+    """
+
     def __init__(self, symbol) -> None:
         super().__init__()
-        # Always GTC and limit orders
-        # limit/market orders will be decided by matching_engine
-        # PRICE_FILTER decimals
         self.symbol = symbol
         pass