Preliminary use of DataFrame calculations on Accounts.

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = D203
+ignore = D203,W503
 exclude = .venv,.git,__pycache__,.ipynb_checkpoints,docs
 per-file-ignores =
     # imported but unused

bin/show-account-dataframe

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import os.path
+import pandas
+import sys
+import typing
+
+from solana.publickey import PublicKey
+
+sys.path.insert(0, os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..")))
+import mango  # nopep8
+
+parser = argparse.ArgumentParser(description="Display the balances of all group tokens in the current wallet.")
+mango.ContextBuilder.add_command_line_parameters(parser)
+mango.Wallet.add_command_line_parameters(parser)
+parser.add_argument("--address", type=PublicKey,
+                    help="Root address to check (if not provided, the wallet address is used)")
+args: argparse.Namespace = mango.parse_args(parser)
+
+address: typing.Optional[PublicKey] = args.address
+if address is None:
+    wallet = mango.Wallet.from_command_line_parameters_or_raise(args)
+    address = wallet.address
+
+context: mango.Context = mango.ContextBuilder.from_command_line_parameters(args)
+group: mango.Group = mango.Group.load(context)
+cache: mango.Cache = mango.Cache.load(context, group.cache)
+
+address_account_info: typing.Optional[mango.AccountInfo] = mango.AccountInfo.load(context, address)
+if address_account_info is None:
+    raise Exception(f"Could not load account data from address {address}")
+
+mango_accounts: typing.Sequence[mango.Account]
+if len(address_account_info.data) == mango.layouts.MANGO_ACCOUNT.sizeof():
+    mango_accounts = [mango.Account.parse(address_account_info, group, cache)]
+else:
+    mango_accounts = mango.Account.load_all_for_owner(context, address, group)
+
+for account in mango_accounts:
+    print("\n⚠ WARNING! ⚠ This is a work-in-progress and these figures may be wrong!\n")
+    pandas.set_option('display.max_columns', None)
+    pandas.set_option('display.width', None)
+    pandas.set_option('precision', 6)
+
+    open_orders: typing.Dict[str, mango.OpenOrders] = account.load_all_spot_open_orders(context)
+    frame: pandas.DataFrame = account.to_dataframe(group, open_orders, cache)
+    print(frame)
+
+    print("Init Health:", account.init_health(frame))
+    print("Maint Health:", account.maint_health(frame))
+    print("Total Value:", account.total_value(frame))

mango/__init__.py

@@ -42,6 +42,7 @@
 from .client import TransactionException as TransactionException
 from .combinableinstructions import CombinableInstructions as CombinableInstructions
 from .constants import MangoConstants as MangoConstants
+from .constants import DATA_PATH as DATA_PATH
 from .constants import SOL_DECIMAL_DIVISOR as SOL_DECIMAL_DIVISOR
 from .constants import SOL_DECIMALS as SOL_DECIMALS
 from .constants import SOL_MINT_ADDRESS as SOL_MINT_ADDRESS

mango/group.py

@@ -293,9 +293,13 @@ def from_layout(layout: typing.Any, name: str, account_info: AccountInfo, versio
                         raise Exception(f"Cannot find base token or perp market info for index {index}")
                     perp_market = market_lookup.find_by_address(perp_market_info.address)
                     if perp_market is None:
-                        logging.warning(f"Group cannot find base token or perp market for index {index}")
-                    else:
-                        base_instrument = perp_market.base
+                        in_slots += [False]
+                        logging.warning(
+                            f"Group cannot find base token or perp market for index {index} - {perp_market_info}")
+                        continue
+
+                    base_instrument = perp_market.base
+
                 if perp_market_info is not None:
                     perp_lot_size_converter = LotSizeConverter(
                         base_instrument, perp_market_info.base_lot_size, quote_token_bank.token, perp_market_info.quote_lot_size)
@@ -391,11 +395,19 @@ def perp_market_cache_from_cache(self, cache: Cache, token: Instrument) -> typin
         market_cache: MarketCache = self.market_cache_from_cache(cache, token)
         return market_cache.perp_market
 
-    def market_cache_from_cache(self, cache: Cache, instrument: Instrument) -> MarketCache:
-        slot: GroupSlot = self.slot_by_instrument(instrument)
+    def market_cache_from_cache_or_none(self, cache: Cache, instrument: Instrument) -> typing.Optional[MarketCache]:
+        slot: typing.Optional[GroupSlot] = self.slot_by_instrument_or_none(instrument)
+        if slot is None:
+            return None
         instrument_index: int = slot.index
         return cache.market_cache_for_index(instrument_index)
 
+    def market_cache_from_cache(self, cache: Cache, instrument: Instrument) -> MarketCache:
+        market_cache: typing.Optional[MarketCache] = self.market_cache_from_cache_or_none(cache, instrument)
+        if market_cache is not None:
+            return market_cache
+        raise Exception(f"Could not find market cache for instrument {instrument.symbol}")
+
     def fetch_cache(self, context: Context) -> Cache:
         return Cache.load(context, self.cache)
 

mango/lotsizeconverter.py

@@ -35,10 +35,17 @@ def lot_size(self) -> Decimal:
     def tick_size(self) -> Decimal:
         return self.price_lots_to_number(Decimal(1))
 
-    def price_lots_to_number(self, price_lots: Decimal) -> Decimal:
+    def adjust_to_base_decimals(self, value: Decimal) -> Decimal:
         adjusted = 10 ** (self.base.decimals - self.quote.decimals)
+        return value * adjusted
+
+    def adjust_to_quote_decimals(self, value: Decimal) -> Decimal:
+        adjusted = 10 ** (self.quote.decimals - self.base.decimals)
+        return value * adjusted
+
+    def price_lots_to_number(self, price_lots: Decimal) -> Decimal:
         lots_to_native = self.quote_lot_size / self.base_lot_size
-        return (price_lots * lots_to_native) * adjusted
+        return self.adjust_to_base_decimals(price_lots * lots_to_native)
 
     def price_number_to_lots(self, price: Decimal) -> int:
         base_factor: Decimal = 10 ** self.base.decimals

mango/openorders.py

@@ -77,14 +77,15 @@ def from_layout(layout: typing.Any, account_info: AccountInfo,
         base_token_total: Decimal = layout.base_token_total / base_divisor
         quote_token_free: Decimal = layout.quote_token_free / quote_divisor
         quote_token_total: Decimal = layout.quote_token_total / quote_divisor
+        referrer_rebate_accrued: Decimal = layout.referrer_rebate_accrued / quote_divisor
 
         placed_orders: typing.Sequence[PlacedOrder] = []
         if account_flags.initialized:
             placed_orders = PlacedOrder.build_from_open_orders_data(
                 layout.free_slot_bits, layout.is_bid_bits, layout.orders, layout.client_ids)
         return OpenOrders(account_info, Version.UNSPECIFIED, program_address, account_flags, layout.market,
                           layout.owner, base_token_free, base_token_total, quote_token_free,
-                          quote_token_total, placed_orders, layout.referrer_rebate_accrued)
+                          quote_token_total, placed_orders, referrer_rebate_accrued)
 
     @staticmethod
     def parse(account_info: AccountInfo, base_decimals: Decimal, quote_decimals: Decimal) -> "OpenOrders":
@@ -148,7 +149,7 @@ def __str__(self) -> str:
     Owner: {self.owner}
     Base Token: {self.base_token_free:,.8f} of {self.base_token_total:,.8f}
     Quote Token: {self.quote_token_free:,.8f} of {self.quote_token_total:,.8f}
-    Referrer Rebate Accrued: {self.referrer_rebate_accrued}
+    Referrer Rebate Accrued: {self.referrer_rebate_accrued:,.8f}
     Orders:
         {placed_orders}
 »"""

mango/perpaccount.py

@@ -77,42 +77,52 @@ def empty(self) -> bool:
         return False
 
     def unsettled_funding(self, perp_market_cache: PerpMarketCache) -> Decimal:
-        if self.base_position < 0:
-            return self.base_position * (perp_market_cache.short_funding - self.short_settled_funding)
+        base_position: Decimal = self.base_position
+        unsettled: Decimal
+        if base_position < 0:
+            unsettled = base_position * (perp_market_cache.short_funding - self.short_settled_funding)
         else:
-            return self.base_position * (perp_market_cache.long_funding - self.long_settled_funding)
+            unsettled = base_position * (perp_market_cache.long_funding - self.long_settled_funding)
+        return - self.lot_size_converter.quote.shift_to_decimals(unsettled)
 
     def asset_value(self, perp_market_cache: PerpMarketCache, price: Decimal) -> Decimal:
+        base_position: Decimal = self.lot_size_converter.adjust_to_quote_decimals(self.base_position)
         value: Decimal = Decimal(0)
-        if self.base_position > 0:
-            value = self.base_position * self.lot_size_converter.base_lot_size * price
-
-        quote_position: Decimal = self.quote_position
-        if self.base_position > 0:
-            quote_position -= (perp_market_cache.long_funding - self.long_settled_funding) * self.base_position
-        elif self.base_position < 0:
-            quote_position -= (perp_market_cache.short_funding - self.short_settled_funding) * self.base_position
+        if base_position > 0:
+            value = base_position * self.lot_size_converter.base_lot_size * price
+            value = self.lot_size_converter.quote.shift_to_decimals(value)
 
+        quote_position: Decimal = self.lot_size_converter.quote.shift_to_decimals(self.quote_position)
+        quote_position += self.unsettled_funding(perp_market_cache)
         if quote_position > 0:
             value += quote_position
 
-        return self.lot_size_converter.quote.shift_to_decimals(value)
+        return value
 
     def liability_value(self, perp_market_cache: PerpMarketCache, price: Decimal) -> Decimal:
+        base_position: Decimal = self.lot_size_converter.adjust_to_quote_decimals(self.base_position)
         value: Decimal = Decimal(0)
-        if self.base_position < 0:
-            value = self.base_position * self.lot_size_converter.base_lot_size * price
-
-        quote_position: Decimal = self.quote_position
-        if self.base_position > 0:
-            quote_position -= (perp_market_cache.long_funding - self.long_settled_funding) * self.base_position
-        elif self.base_position < 0:
-            quote_position -= (perp_market_cache.short_funding - self.short_settled_funding) * self.base_position
+        if base_position < 0:
+            value = base_position * self.lot_size_converter.base_lot_size * price
+            value = self.lot_size_converter.quote.shift_to_decimals(value)
 
+        quote_position: Decimal = self.lot_size_converter.quote.shift_to_decimals(self.quote_position)
+        quote_position += self.unsettled_funding(perp_market_cache)
         if quote_position < 0:
             value += quote_position
 
-        return self.lot_size_converter.quote.shift_to_decimals(-value)
+        return value
+
+    def current_value(self, perp_market_cache: PerpMarketCache, price: Decimal) -> Decimal:
+        base_position: Decimal = self.lot_size_converter.adjust_to_quote_decimals(self.base_position)
+        value: Decimal = base_position * self.lot_size_converter.base_lot_size * price
+
+        quote_position: Decimal = self.lot_size_converter.quote.shift_to_decimals(self.quote_position)
+        quote_position += self.unsettled_funding(perp_market_cache)
+
+        value += quote_position
+
+        return self.lot_size_converter.quote.shift_to_decimals(value)
 
     def __str__(self) -> str:
         if self.empty:

tests/calculations/test_healthcalculator.py

@@ -25,47 +25,46 @@ def test_1deposit() -> None:
     assert health == Decimal("37904.2600000591928892771752953600134")
 
 
-def test_perp_account_no_spot_openorders() -> None:
+def test_account1() -> None:
     context = fake_context()
-    group, cache, account, open_orders = load_data_from_directory("tests/testdata/perp_account_no_spot_openorders")
+    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account1")
 
     actual = HealthCalculator(context, HealthType.INITIAL)
     health = actual.calculate(account, open_orders, group, cache)
-
-    # Typescript says: 341025333625.51856223547208912805
+    # Typescript says: 454884281.15520619643754685058
     # TODO: This is significantly different from Typescript answer
-    assert health == Decimal("7036880.69722812395986194177339495613")
+    assert health == Decimal("2578453.62435039273978679178827388534")
 
 
-def test_perp_account_no_spot_openorders_unhealthy() -> None:
+def test_account2() -> None:
     context = fake_context()
-    group, cache, account, open_orders = load_data_from_directory(
-        "tests/testdata/perp_account_no_spot_openorders_unhealthy")
+    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account2")
 
     actual = HealthCalculator(context, HealthType.INITIAL)
     health = actual.calculate(account, open_orders, group, cache)
-    # Typescript says: -848086876487.04950427436299875694
-    # TODO: This is significantly different from Typescript answer
-    assert health == Decimal("1100318.49506000114695611699892507857")
+    # Typescript says: 7516159604.84918334545095675026
+    # TODO: This is slightly different from Typescript answer
+    assert health == Decimal("-34471.8822627460347363357247598728190")
 
 
-def test_account1() -> None:
+def test_account3() -> None:
     context = fake_context()
-    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account1")
+    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account3")
 
     actual = HealthCalculator(context, HealthType.INITIAL)
     health = actual.calculate(account, open_orders, group, cache)
-    # Typescript says: 454884281.15520619643754685058
+
+    # Typescript says: 341025333625.51856223547208912805
     # TODO: This is significantly different from Typescript answer
-    assert health == Decimal("2578453.62435039273978679178827388534")
+    assert health == Decimal("7036880.69722812395986194177339495613")
 
 
-def test_account2() -> None:
+def test_account4() -> None:
     context = fake_context()
-    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account2")
+    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account4")
 
     actual = HealthCalculator(context, HealthType.INITIAL)
     health = actual.calculate(account, open_orders, group, cache)
-    # Typescript says: 7516159604.84918334545095675026
-    # TODO: This is slightly different from Typescript answer
-    assert health == Decimal("-34471.8822627460347363357247598728190")
+    # Typescript says: -848086876487.04950427436299875694
+    # TODO: This is significantly different from Typescript answer
+    assert health == Decimal("1100318.49506000114695611699892507857")

tests/data.py

@@ -9,9 +9,13 @@ def load_group(filename: str) -> mango.Group:
     account_info: mango.AccountInfo = mango.AccountInfo.load_json(filename)
     mainnet_token_lookup: mango.InstrumentLookup = mango.IdsJsonTokenLookup("mainnet", "mainnet.1")
     devnet_token_lookup: mango.InstrumentLookup = mango.IdsJsonTokenLookup("devnet", "devnet.2")
+    devnet_non_spl_instrument_lookup: mango.InstrumentLookup = mango.NonSPLInstrumentLookup.load(
+        mango.NonSPLInstrumentLookup.DefaultDevnetDataFilepath)
     instrument_lookup: mango.InstrumentLookup = mango.CompoundInstrumentLookup(
-        [mainnet_token_lookup, devnet_token_lookup])
-    market_lookup: mango.MarketLookup = mango.NullMarketLookup()
+        [mainnet_token_lookup, devnet_token_lookup, devnet_non_spl_instrument_lookup])
+    mainnet_market_lookup: mango.MarketLookup = mango.IdsJsonMarketLookup("mainnet", instrument_lookup)
+    devnet_market_lookup: mango.MarketLookup = mango.IdsJsonMarketLookup("devnet", instrument_lookup)
+    market_lookup: mango.MarketLookup = mango.CompoundMarketLookup([mainnet_market_lookup, devnet_market_lookup])
     return mango.Group.parse(account_info, "devnet.2", instrument_lookup, market_lookup)
 
 

tests/test_account.py

@@ -1,6 +1,7 @@
 import pytest
 
 from .context import mango
+from .data import load_data_from_directory
 from .fakes import fake_account_info, fake_seeded_public_key, fake_token_bank, fake_instrument, fake_instrument_value, fake_perp_account, fake_token
 
 from decimal import Decimal
@@ -214,3 +215,64 @@ def test_slot_lookups() -> None:
     assert actual.slot_by_instrument(fake_instrument("slot3")) == slots[2]
     with pytest.raises(Exception):
         assert actual.slot_by_instrument(fake_instrument())
+
+
+def test_loaded_account_slot_lookups() -> None:
+    group, cache, account, open_orders = load_data_from_directory("tests/testdata/account5")
+    assert len(account.slots) == 14
+
+    # « GroupSlot[0] « Token [MNGO] 'MNGO' [Bb9bsTQa1bGEtQ5KagGkvSHyuLqDWumFUcRqFusFNJWC (6 decimals)] »
+    assert account.slots_by_index[0] is not None
+    assert account.slots_by_index[0].base_instrument.symbol == "MNGO"
+
+    # « GroupSlot[1] « Token [BTC] 'BTC' [3UNBZ6o52WTWwjac2kPUb4FyodhU1vFkRJheu1Sh2TvU (6 decimals)] »
+    assert account.slots_by_index[1] is not None
+    assert account.slots_by_index[1].base_instrument.symbol == "BTC"
+
+    # « GroupSlot[2] « Token [ETH] 'ETH' [Cu84KB3tDL6SbFgToHMLYVDJJXdJjenNzSKikeAvzmkA (6 decimals)] »
+    assert account.slots_by_index[2] is not None
+    assert account.slots_by_index[2].base_instrument.symbol == "ETH"
+
+    # « GroupSlot[3] « Token [SOL] 'SOL' [So11111111111111111111111111111111111111112 (9 decimals)] »
+    assert account.slots_by_index[3] is not None
+    assert account.slots_by_index[3].base_instrument.symbol == "SOL"
+
+    # « GroupSlot[4] « Token [SRM] 'SRM' [AvtB6w9xboLwA145E221vhof5TddhqsChYcx7Fy3xVMH (6 decimals)] »
+    assert account.slots_by_index[4] is not None
+    assert account.slots_by_index[4].base_instrument.symbol == "SRM"
+
+    # « GroupSlot[5] « Token [RAY] 'RAY' [3YFQ7UYJ7sNGpXTKBxM3bYLVxKpzVudXAe4gLExh5b3n (6 decimals)] »
+    assert account.slots_by_index[5] is not None
+    assert account.slots_by_index[5].base_instrument.symbol == "RAY"
+
+    # « GroupSlot[6] « Token [USDT] 'USDT' [DAwBSXe6w9g37wdE2tCrFbho3QHKZi4PjuBytQCULap2 (6 decimals)] »
+    assert account.slots_by_index[6] is not None
+    assert account.slots_by_index[6].base_instrument.symbol == "USDT"
+
+    # « GroupSlot[7] « Instrument [ADA] 'Cardano' »
+    assert account.slots_by_index[7] is not None
+    assert account.slots_by_index[7].base_instrument.symbol == "ADA"
+
+    # « GroupSlot[8] « Token [FTT] 'FTT' [Fxh4bpZnRCnpg2vcH11ttmSTDSEeC5qWbPRZNZWnRnqY (6 decimals)] »
+    assert account.slots_by_index[8] is not None
+    assert account.slots_by_index[8].base_instrument.symbol == "FTT"
+
+    # « GroupSlot[9] « Instrument [AVAX] 'Avalanche' »
+    assert account.slots_by_index[9] is not None
+    assert account.slots_by_index[9].base_instrument.symbol == "AVAX"
+
+    # « GroupSlot[10] « Instrument [LUNA] 'Terra' »
+    assert account.slots_by_index[10] is not None
+    assert account.slots_by_index[10].base_instrument.symbol == "LUNA"
+
+    # « GroupSlot[11] « Instrument [BNB] 'Binance Coin' »
+    assert account.slots_by_index[11] is not None
+    assert account.slots_by_index[11].base_instrument.symbol == "BNB"
+
+    # « GroupSlot[12] « Instrument [MATIC] 'Polygon' »
+    assert account.slots_by_index[12] is not None
+    assert account.slots_by_index[12].base_instrument.symbol == "MATIC"
+    assert account.slots_by_index[13] is None
+    assert account.slots_by_index[14] is None
+    assert account.slots_by_index[15] is not None
+    assert account.slots_by_index[15].base_instrument.symbol == "USDC"