diff --git a/api/account/assets.py b/api/account/assets.py index 03fbcacb7..a2c7d8c33 100644 --- a/api/account/assets.py +++ b/api/account/assets.py @@ -15,7 +15,6 @@ class Assets(BaseDeal): def __init__(self): - self.db = Database() self.usd_balance = 0 self.exception_list = [] @@ -444,45 +443,7 @@ def get_market_domination(self, size=7): Returns: dict: A dictionary containing the market domination data, including gainers and losers counts, percentages, and dates. """ - try: - data = self.db.query_market_domination(size=size) - market_domination_series = MarketDominationSeries() - - for item in data: - gainers_percent = 0 - losers_percent = 0 - gainers_count = 0 - losers_count = 0 - total_volume = 0 - if "data" in item: - for crypto in item["data"]: - if float(crypto['priceChangePercent']) > 0: - gainers_percent += float(crypto['volume']) - gainers_count += 1 - - if float(crypto['priceChangePercent']) < 0: - losers_percent += abs(float(crypto['volume'])) - losers_count += 1 - - if float(crypto['volume']) > 0: - total_volume += float(crypto['volume']) * float(crypto['price']) - - market_domination_series.dates.append(item["time"]) - market_domination_series.gainers_percent.append(gainers_percent) - market_domination_series.losers_percent.append(losers_percent) - market_domination_series.gainers_count.append(gainers_count) - market_domination_series.losers_count.append(losers_count) - market_domination_series.total_volume.append(total_volume) - - market_domination_series.dates = market_domination_series.dates[-size:] - market_domination_series.gainers_percent = market_domination_series.gainers_percent[-size:] - market_domination_series.losers_percent = market_domination_series.losers_percent[-size:] - market_domination_series.gainers_count = market_domination_series.gainers_count[-size:] - market_domination_series.losers_count = market_domination_series.losers_count[-size:] - market_domination_series.total_volume = market_domination_series.total_volume[-size:] - - data = market_domination_series.model_dump(mode="json") - - return json_response({ "data": data, "message": "Successfully retrieved market domination data.", "error": 0 }) - except Exception as error: - return json_response_error(f"Failed to retrieve market domination data: {error}") + query = {"$query": {}, "$orderby": {"_id": -1}} + result = self._db.market_domination.find(query).limit(size) + return list(result) + diff --git a/api/account/routes.py b/api/account/routes.py index 85bdb6a7b..440f5ebd2 100644 --- a/api/account/routes.py +++ b/api/account/routes.py @@ -2,7 +2,15 @@ from datetime import datetime, timedelta from account.account import Account from account.assets import Assets -from account.schemas import BalanceResponse, GainersLosersResponse, BalanceSeriesResponse, GetMarketDominationResponse, MarketDominationResponse +from account.schemas import ( + BalanceResponse, + GainersLosersResponse, + BalanceSeriesResponse, + GetMarketDominationResponse, + MarketDominationResponse, + MarketDominationSeries, +) +from tools.handle_error import json_response, json_response_error account_blueprint = APIRouter() @@ -70,32 +78,112 @@ def store_balance(): return Assets().store_balance() -@account_blueprint.get("/gainers-losers", response_model=GainersLosersResponse, tags=["assets"]) +@account_blueprint.get( + "/gainers-losers", response_model=GainersLosersResponse, tags=["assets"] +) async def retrieve_gainers_losers(): return await Assets().retrieve_gainers_losers() -@account_blueprint.get("/balance-series", response_model=BalanceSeriesResponse, tags=["assets"]) + +@account_blueprint.get( + "/balance-series", response_model=BalanceSeriesResponse, tags=["assets"] +) async def get_balance_series(): today = datetime.now() month_ago = today - timedelta(30) - return await Assets().get_balance_series(start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today)) + return await Assets().get_balance_series( + start_date=datetime.timestamp(month_ago), end_date=datetime.timestamp(today) + ) + @account_blueprint.get("/clean", response_model=BalanceSeriesResponse, tags=["assets"]) def clean_balance(): return Assets().clean_balance_assets() -@account_blueprint.get("/disable-isolated", response_model=BalanceSeriesResponse, tags=["assets"]) + +@account_blueprint.get( + "/disable-isolated", response_model=BalanceSeriesResponse, tags=["assets"] +) def disable_isolated(): return Assets().disable_isolated_accounts() + @account_blueprint.get("/one-click-liquidation/{asset}", tags=["assets"]) def one_click_liquidation(asset): return Assets().one_click_liquidation(asset) -@account_blueprint.get("/market-domination", tags=["assets"], response_model=MarketDominationResponse) -def market_domination(size: int=7): - return Assets().get_market_domination(size) -@account_blueprint.get("/store-market-domination", tags=["assets"], response_model=GetMarketDominationResponse) +@account_blueprint.get( + "/market-domination", tags=["assets"], response_model=MarketDominationResponse +) +def market_domination(size: int = 7): + + data = Assets().get_market_domination(size) + market_domination_series = MarketDominationSeries() + + try: + for item in data: + gainers_percent = 0 + losers_percent = 0 + gainers_count = 0 + losers_count = 0 + total_volume = 0 + if "data" in item: + for crypto in item["data"]: + if float(crypto['priceChangePercent']) > 0: + gainers_percent += float(crypto['volume']) + gainers_count += 1 + + if float(crypto['priceChangePercent']) < 0: + losers_percent += abs(float(crypto['volume'])) + losers_count += 1 + + if float(crypto['volume']) > 0: + total_volume += float(crypto['volume']) * float(crypto['price']) + + market_domination_series.dates.append(item["time"]) + market_domination_series.gainers_percent.append(gainers_percent) + market_domination_series.losers_percent.append(losers_percent) + market_domination_series.gainers_count.append(gainers_count) + market_domination_series.losers_count.append(losers_count) + market_domination_series.total_volume.append(total_volume) + + market_domination_series.dates = market_domination_series.dates[-size:] + market_domination_series.gainers_percent = ( + market_domination_series.gainers_percent[-size:] + ) + market_domination_series.losers_percent = ( + market_domination_series.losers_percent[-size:] + ) + market_domination_series.gainers_count = market_domination_series.gainers_count[ + -size: + ] + market_domination_series.losers_count = market_domination_series.losers_count[ + -size: + ] + market_domination_series.total_volume = market_domination_series.total_volume[ + -size: + ] + + data = market_domination_series.model_dump(mode="json") + + return json_response( + { + "data": data, + "message": "Successfully retrieved market domination data.", + "error": 0, + } + ) + except Exception as error: + return json_response_error( + f"Failed to retrieve market domination data: {error}" + ) + + +@account_blueprint.get( + "/store-market-domination", + tags=["assets"], + response_model=GetMarketDominationResponse, +) def store_market_domination(): return Assets().store_market_domination() diff --git a/api/db.py b/api/db.py index ac5de6e33..45a87f714 100644 --- a/api/db.py +++ b/api/db.py @@ -16,28 +16,20 @@ def setup_db(): class Database: - def __init__(self): - self._db = setup_db() - - def get_autotrade_settings(self): - return self._db.research_controller.find_one({"_id": "settings"}) - - def get_test_autotrade_settings(self): - return self._db.research_controller.find_one({"_id": "test_autotrade_settings"}) - - def get_active_bots(self): - bots = list(self._db.bots.distinct("pair", {"status": "active"})) - return bots - - def get_active_paper_trading_bots(self): - bots = list(self._db.paper_trading.distinct("pair", {"status": "active"})) - return bots - - def query_market_domination(self, size: int = 7): - """ - Get market domination data and order by DESC _id, - which means earliest data will be at the end of the list. - """ - query = {"$query": {}, "$orderby": {"_id": -1}} - result = self._db.market_domination.find(query).limit(size) - return list(result) + _db = setup_db() + + + # def get_autotrade_settings(self): + # return self._db.research_controller.find_one({"_id": "settings"}) + + # def get_test_autotrade_settings(self): + # return self._db.research_controller.find_one({"_id": "test_autotrade_settings"}) + + # def get_active_bots(self): + # bots = list(self._db.bots.distinct("pair", {"status": "active"})) + # return bots + + # def get_active_paper_trading_bots(self): + # bots = list(self._db.paper_trading.distinct("pair", {"status": "active"})) + # return bots + diff --git a/api/deals/base.py b/api/deals/base.py index ae2760fd3..301986cfc 100644 --- a/api/deals/base.py +++ b/api/deals/base.py @@ -4,7 +4,7 @@ import logging from time import time -from db import setup_db +from db import Database, setup_db from orders.controller import OrderController from bots.schemas import BotSchema from pymongo import ReturnDocument @@ -16,7 +16,7 @@ from tools.enum_definitions import Status, Strategy from bson.objectid import ObjectId from deals.schema import DealSchema, OrderSchema -from datetime import datetime, timedelta +from datetime import datetime # To be removed one day when commission endpoint found that provides this value ESTIMATED_COMMISSIONS_RATE = 0.0075 @@ -27,11 +27,10 @@ class BaseDeal(OrderController): """ def __init__(self, bot, db_collection_name): - self.active_bot = BotSchema.parse_obj(bot) + self.active_bot = BotSchema(**bot) self.symbol = self.active_bot.pair super().__init__(symbol=self.active_bot.pair) - self.db = setup_db() - self.db_collection = self.db[db_collection_name] + self.db_collection = self._db[db_collection_name] self.market_domination_reversal = None if self.active_bot.strategy == Strategy.margin_short: self.isolated_balance: float = self.get_isolated_balance(self.symbol) @@ -174,22 +173,6 @@ def close_open_orders(self, symbol): return True return False - def update_required(self): - """ - Terminate streaming and restart list of bots required - - This will queue up a timer to restart streaming_controller when timer is reached - This timer is added, so that update_required petitions can be accumulated and - avoid successively restarting streaming_controller, which consumes a lot of memory - - This means that everytime there is an update in the list of active bots, - it will reset the timer - """ - self.db.research_controller.update_one( - {"_id": "settings"}, {"$set": {"update_required": time()}} - ) - return - def save_bot_streaming(self): """ MongoDB query to save bot using Pydantic @@ -234,10 +217,7 @@ def create_new_bot_streaming(self): bot = encode_json(self.active_bot) self.db_collection.insert_one(bot) new_bot = self.db_collection.find_one({"id": bot["id"]}) - bot_class = BotSchema.parse_obj(new_bot) - - # notify the system to update streaming as usual bot actions - self.db.research_controller.update_one({"_id": "settings"}, {"$set": {"update_required": time()}}) + bot_class = BotSchema(**new_bot) return bot_class @@ -318,7 +298,7 @@ def base_order(self): return document def dynamic_take_profit(self, current_bot, close_price): - self.active_bot = BotSchema.parse_obj(current_bot) + self.active_bot = BotSchema(**current_bot) params = { "symbol": self.active_bot.pair, diff --git a/api/orders/controller.py b/api/orders/controller.py index 65204aa3f..5721d1a20 100644 --- a/api/orders/controller.py +++ b/api/orders/controller.py @@ -1,6 +1,7 @@ import logging from account.account import Account +from db import Database from tools.enum_definitions import OrderType, TimeInForce, OrderSide from tools.handle_error import json_response, json_response_error, json_response_message from tools.round_numbers import supress_notation @@ -10,12 +11,15 @@ poll_percentage = 0 -class OrderController(Account): +class OrderController(Database, Account): + """ + Always GTC and limit orders + limit/market orders will be decided by matching_engine + PRICE_FILTER decimals + """ + def __init__(self, symbol) -> None: super().__init__() - # Always GTC and limit orders - # limit/market orders will be decided by matching_engine - # PRICE_FILTER decimals self.symbol = symbol pass diff --git a/api/research/controller.py b/api/research/controller.py index a85982940..ee5d08e19 100644 --- a/api/research/controller.py +++ b/api/research/controller.py @@ -1,4 +1,4 @@ -from db import setup_db +from db import Database, setup_db from datetime import datetime from time import sleep from tools.handle_error import json_response, json_response_error, json_response_message @@ -8,7 +8,7 @@ from pymongo import ASCENDING from fastapi.encoders import jsonable_encoder -class Controller: +class Controller(Database): """ Research app settings - Get: get single document with settings @@ -16,24 +16,22 @@ class Controller: """ def __init__(self): - self.db = setup_db() + super().__init__() def get_blacklist(self): """ Get list of blacklisted symbols """ - query_result = self.db.blacklist.find({ "pair": { "$exists": True } }).sort("pair", ASCENDING) + query_result = self._db.blacklist.find({ "pair": { "$exists": True } }).sort("pair", ASCENDING) blacklist = list(query_result) - return json_response( - {"message": "Successfully retrieved blacklist", "data": blacklist} - ) + return blacklist def create_blacklist_item(self, data): try: blacklist_item = data.dict() blacklist_item["_id"] = data.pair - self.db.blacklist.insert_one(blacklist_item) + self._db.blacklist.insert_one(blacklist_item) return json_response( {"message": "Successfully updated blacklist"} ) @@ -45,7 +43,7 @@ def create_blacklist_item(self, data): def delete_blacklist_item(self, pair): - blacklist = self.db.blacklist.delete_one({"_id": pair}) + blacklist = self._db.blacklist.delete_one({"_id": pair}) if blacklist.acknowledged: resp = json_response({"message": "Successfully deleted item from blacklist", "data": str(pair)}) @@ -58,7 +56,7 @@ def edit_blacklist(self, data): if "pair" not in data: return json_response({"message": "Missing required field 'pair'.", "error": 1}) - blacklist = self.db.blacklist.update_one( + blacklist = self._db.blacklist.update_one( {"_id": data["pair"]}, {"$set": data} ) @@ -118,8 +116,8 @@ def store_profitable_signals(self): }) try: - self.db.three_commas_signals.delete_many({}) - self.db.three_commas_signals.insert_many(consolidated_signals) + self._db.three_commas_signals.delete_many({}) + self._db.three_commas_signals.insert_many(consolidated_signals) except Exception as err: print(err) @@ -131,7 +129,7 @@ def get_3commas_signals(self): per week """ query = {} - signals = list(self.db.three_commas_signals.find(query)) + signals = list(self._db.three_commas_signals.find(query)) return json_response({"message": "Successfully retrieved profitable 3commas signals", "data": signals}) @@ -142,14 +140,14 @@ def get_3commas_signals(self): To merge with blacklist """ def get_subscribed_symbols(self): - query_result = self.db.subscribed_symbols.find({}).sort("pair", ASCENDING) + query_result = self._db.subscribed_symbols.find({}).sort("pair", ASCENDING) all_symbols = list(query_result) return json_response( {"message": "Successfully retrieved blacklist", "data": all_symbols} ) def delete_all_subscribed_symbols(self): - query_result = self.db.subscribed_symbols.delete_many({}) + query_result = self._db.subscribed_symbols.delete_many({}) return json_response( {"message": "Successfully deleted all symbols", "data": { @@ -159,9 +157,9 @@ def delete_all_subscribed_symbols(self): def bulk_upsert_all(self, data): symbols = jsonable_encoder(data) - self.db.subscribed_symbols.delete_many({}) + self._db.subscribed_symbols.delete_many({}) try: - query_result = self.db.subscribed_symbols.insert_many( + query_result = self._db.subscribed_symbols.insert_many( symbols, ) return json_response( @@ -175,7 +173,7 @@ def bulk_upsert_all(self, data): def edit_subscribed_symbol(self, symbol): symbol = jsonable_encoder(symbol) try: - self.db.subscribed_symbols.update_one( + self._db.subscribed_symbols.update_one( symbol, upsert=True, ) diff --git a/api/research/routes.py b/api/research/routes.py index ec1b11948..97d9f391f 100644 --- a/api/research/routes.py +++ b/api/research/routes.py @@ -3,6 +3,7 @@ from apis import ThreeCommasApi from research.controller import Controller from research.schemas import BlacklistSchema, BlacklistResponse, SubscribedSymbolsSchema +from tools.handle_error import json_response, json_response_error, json_response_message research_blueprint = APIRouter() @@ -31,12 +32,15 @@ def put_blacklist(blacklist_item: BlacklistSchema): return Controller().edit_blacklist(blacklist_item) -@research_blueprint.get("/blacklist", response_model=BlacklistResponse, tags=["blacklist and research"]) +@research_blueprint.get( + "/blacklist", response_model=BlacklistResponse, tags=["blacklist and research"] +) def get_blacklisted(): """ Get all symbols/pairs blacklisted """ - return Controller().get_blacklist() + data = Controller().get_blacklist() + return json_response({"message": "Successfully retrieved blacklist", "data": data}) @research_blueprint.get("/3commas-presets", tags=["blacklist and research"]) @@ -53,6 +57,7 @@ def get_subscribed_symbols(): def create_subscribed_symbols(data: list[SubscribedSymbolsSchema]): return Controller().bulk_upsert_all(data) + @research_blueprint.put("/subscribed/{symbol}", tags=["blacklist and research"]) def edit_subscribed_symbol(symbol: str): return Controller().edit_subscribed_symbol(symbol) diff --git a/api/tests/test_assets.py b/api/tests/test_assets.py index 6ece4da73..595d16642 100644 --- a/api/tests/test_assets.py +++ b/api/tests/test_assets.py @@ -15,51 +15,13 @@ @pytest.fixture() def patch_database(monkeypatch): - data = [ - { - '_id': '65d39b44a7ff2a5e843b1e0b', - 'time': '2024-02-19 18:17:40.479', - 'data': [ - { - 'symbol': 'PIXELUSDT', - 'priceChangePercent': '15.000', - 'volume': '1235816253.30000000', - 'price': '0.64880000' - }, - { - 'symbol': 'PERPUSDT', - 'priceChangePercent': '20.596', - 'volume': '24276162.79000000', - 'price': '1.48359000' - }, - ] - }, - { - '_id': '65d39b44a7ff2a5e843b1e0b', - 'time': '2024-02-19 18:17:40.479', - 'data': [ - { - 'symbol': 'PIXELUSDT', - 'priceChangePercent': '1522.000', - 'volume': '1235816253.30000000', - 'price': '0.64880000' - }, - { - 'symbol': 'PERPUSDT', - 'priceChangePercent': '20.596', - 'volume': '24276162.79000000', - 'price': '1.48359000' - }, - ] - } - ] + data = [{'_id': '65d39b44a7ff2a5e843b1e0b', 'time': '2024-02-19 18:17:40.479', 'data': [{'symbol': 'PIXELUSDT', 'priceChangePercent': '15.000', 'volume': '1235816253.30000000', 'price': '0.64880000'}]}] def new_init(self): self._db = db - monkeypatch.setattr(Database, "__init__", new_init) - monkeypatch.setattr(Database, "query_market_domination", lambda a, size, *arg: data) - return data + monkeypatch.setattr(Assets, "_db", new_init) + monkeypatch.setattr(Assets, "get_market_domination", lambda a, size, *arg: data) def test_get_market_domination(monkeypatch, patch_database): @@ -69,13 +31,12 @@ def test_get_market_domination(monkeypatch, patch_database): # Assert the expected result expected_result = { "data": { - "dates": ['2024-02-19 18:17:40.479' -, "2024-02-19 18:17:40.479"], - "gainers_percent": [1260092416.09, 1260092416.09], - "losers_percent": [0.0, 0.0], - "gainers_count": [2, 2], - "losers_count": [0, 0], - "total_volume": [837813457.4946561, 837813457.4946561] + "dates": ['2024-02-19 18:17:40.479'], + "gainers_percent": [1235816253.3], + "losers_percent": [0.0], + "gainers_count": [1], + "losers_count": [0], + "total_volume": [801797585.14104] }, "message": "Successfully retrieved market domination data.", "error": 0, diff --git a/api/tests/test_research.py b/api/tests/test_research.py new file mode 100644 index 000000000..026482614 --- /dev/null +++ b/api/tests/test_research.py @@ -0,0 +1,54 @@ +from fastapi.testclient import TestClient +import pytest +from research.controller import Controller +from main import app +import mongomock + + +@pytest.fixture() +def mock_research(monkeypatch): + blacklist = [ + { + "_id": "65d39b44a7ff2a5e843b1e0b", + "time": "2024-02-19 18:17:40.479", + "data": [ + { + "_id": "1INCHDOWNUSDT", + "pair": "1INCHDOWNUSDT", + "reason": "Fiat coin or Margin trading", + } + ], + } + ] + + def new_init(self): + mongo_client = mongomock.MongoClient() + self._db = mongo_client.db + + monkeypatch.setattr(Controller, "_db", new_init) + monkeypatch.setattr(Controller, "get_blacklist", lambda self: blacklist) + + +def test_get_blacklisted(mock_research): + client = TestClient(app) + response = client.get("/research/blacklist") + + # Assert the expected result + expected_result = { + "message": "Successfully retrieved blacklist", + "data": [ + { + "_id": "65d39b44a7ff2a5e843b1e0b", + "time": "2024-02-19 18:17:40.479", + "data": [ + { + "_id": "1INCHDOWNUSDT", + "pair": "1INCHDOWNUSDT", + "reason": "Fiat coin or Margin trading", + } + ], + } + ], + } + assert response.status_code == 200 + assert response.json() == expected_result diff --git a/api/tools/enum_definitions.py b/api/tools/enum_definitions.py index 91c865012..98e375ea1 100644 --- a/api/tools/enum_definitions.py +++ b/api/tools/enum_definitions.py @@ -93,4 +93,10 @@ class CloseConditions(str, Enum): market_reversal = "market_reversal" # binbot-research param (self.market_trend_reversal) def __str__(self): - return str(self.str) \ No newline at end of file + return str(self.str) + + +class TrendEnum(str, Enum): + up_trend = "uptrend" + down_trend = "downtrend" + neutral = None \ No newline at end of file diff --git a/binquant b/binquant index 214758aa1..f73ee607c 160000 --- a/binquant +++ b/binquant @@ -1 +1 @@ -Subproject commit 214758aa179d78ea4fef452855eb38e66d83a574 +Subproject commit f73ee607ce8f72e57b082590adafe61c2c7788c2 diff --git a/docker-compose.yml b/docker-compose.yml index c89289f71..6d0151056 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,9 +40,10 @@ services: - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_CFG_NODE_ID=1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true + - KAFKA_CFG_NUM_PARTITIONS=950 - BITNAMI_DEBUG=yes healthcheck: - test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server kafka:9092 --topic hc --describe"] + test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server kafka:9092 --topic klines_store_topic --create --if-not-exists", "kafka-topics.sh --bootstrap-server kafka:9092 --topic klines_store_topic --describe"] start_period: 10s interval: 5s timeout: 10s