Remove additional day (#510)

- remove extra day in index - calculate last possible market opening and schedule opening if before that - fix error in handle_output_message when content is None Co-authored-by: Florian Maurer <[email protected]>
assume-framework · Dec 5, 2024 · 7457a21 · 7457a21
1 parent 44fb0fc
commit 7457a21
Show file tree

Hide file tree

Showing 11 changed files with 110 additions and 58 deletions.
diff --git a/assume/common/outputs.py b/assume/common/outputs.py
@@ -227,7 +227,7 @@ def handle_output_message(self, content: dict, meta: MetaDict):
         content_type = content.get("type")
         market_id = content.get("market_id")
 
-        if not content_data:
+        if content_data is None or len(content_data) == 0:
             return
 
         if content_type in [
@@ -404,7 +404,9 @@ def convert_flows(self, data: dict[tuple[datetime, str], float]):
         if isinstance(data, pd.DataFrame):
             df = data
 
-        # if data is dict
+        # if data is list
+        elif isinstance(data, list):
+            df = pd.DataFrame.from_dict(data)
         elif isinstance(data, dict):
             # Convert the dictionary to a DataFrame
             df = pd.DataFrame.from_dict(
@@ -557,6 +559,7 @@ def create_line(row):
                 continue
             df["simulation"] = self.simulation_id
             df.reset_index()
+            df.columns = df.columns.str.lower()
 
             try:
                 with self.db.begin() as db:

diff --git a/assume/markets/base_market.py b/assume/markets/base_market.py
@@ -49,10 +49,17 @@ class MarketMechanism:
     def __init__(self, marketconfig: MarketConfig):
         super().__init__()
         self.marketconfig = marketconfig
+        # calculate last possible market opening as the difference between the market end
+        # and the length of the longest product plus the delivery time of the products
+        self.last_market_opening = marketconfig.opening_hours._until - max(
+            market_product.duration * market_product.count
+            + market_product.first_delivery
+            for market_product in marketconfig.market_products
+        )
 
     def clear(
         self, orderbook: Orderbook, market_products: list[MarketProduct]
-    ) -> tuple[Orderbook, Orderbook, list[dict]]:
+    ) -> tuple[Orderbook, Orderbook, list[dict], dict[tuple, float]]:
         """
         Clears the market.
 
@@ -263,7 +270,7 @@ async def opening(self):
 
         # schedule the next opening too
         next_opening = self.marketconfig.opening_hours.after(market_open)
-        if next_opening:
+        if next_opening <= self.last_market_opening:
             next_opening_ts = datetime2timestamp(next_opening)
             self.context.schedule_timestamp_task(self.opening(), next_opening_ts)
             logger.debug(

diff --git a/assume/markets/clearing_algorithms/nodal_pricing.py b/assume/markets/clearing_algorithms/nodal_pricing.py
@@ -97,7 +97,7 @@ def setup(self):
 
     def clear(
         self, orderbook: Orderbook, market_products
-    ) -> tuple[Orderbook, Orderbook, list[dict]]:
+    ) -> tuple[Orderbook, Orderbook, list[dict], dict[tuple, float]]:
         """
         Clears the market by running a linear optimal power flow (LOPF) with PyPSA.
 
@@ -109,6 +109,8 @@ def clear(
             Tuple[Orderbook, Orderbook, List[dict]]: The accepted orderbook, rejected orderbook and market metadata.
         """
 
+        if len(orderbook) <= 0:
+            return super().clear(orderbook, market_products)
         orderbook_df = pd.DataFrame(orderbook)
         orderbook_df["accepted_volume"] = 0.0
         orderbook_df["accepted_price"] = 0.0

diff --git a/assume/markets/clearing_algorithms/redispatch.py b/assume/markets/clearing_algorithms/redispatch.py
@@ -95,7 +95,7 @@ def setup(self):
 
     def clear(
         self, orderbook: Orderbook, market_products
-    ) -> tuple[Orderbook, Orderbook, list[dict]]:
+    ) -> tuple[Orderbook, Orderbook, list[dict], dict[tuple, float]]:
         """
         Performs redispatch to resolve congestion in the electricity market.
         It first checks for congestion in the network and if it finds any, it performs redispatch to resolve it.
@@ -110,6 +110,8 @@ def clear(
             Tuple[Orderbook, Orderbook, List[dict]]: The accepted orderbook, rejected orderbook and market metadata.
         """
 
+        if len(orderbook) == 0:
+            return super().clear(orderbook, market_products)
         orderbook_df = pd.DataFrame(orderbook)
         orderbook_df["accepted_volume"] = 0.0
         orderbook_df["accepted_price"] = 0.0

diff --git a/assume/scenario/loader_csv.py b/assume/scenario/loader_csv.py
@@ -5,7 +5,7 @@
 import copy
 import logging
 from collections import defaultdict
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 
 import dateutil.rrule as rr
@@ -439,7 +439,7 @@ def load_config_and_create_forecaster(
 
     index = pd.date_range(
         start=start,
-        end=end + timedelta(days=1),
+        end=end,
         freq=config["time_step"],
     )
 

diff --git a/assume/world.py b/assume/world.py
@@ -44,7 +44,7 @@
 stdout_handler = logging.StreamHandler(stream=sys.stdout)
 handlers = [file_handler, stdout_handler]
 logging.basicConfig(level=logging.INFO, handlers=handlers)
-logging.getLogger("mango").setLevel(logging.WARNING)
+logging.getLogger("mango").setLevel(logging.ERROR)
 
 logger = logging.getLogger(__name__)
 

diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
@@ -41,6 +41,10 @@ Upcoming Release
   - **Tutorials**: General fixes of the tutorials, to align with updated functionalitites of Assume
   - **Tutorial 07**: Aligned Amiris loader with changes in format in Amiris compare (https://gitlab.com/fame-framework/fame-io/-/issues/203 and https://gitlab.com/fame-framework/fame-io/-/issues/208)
   - **Powerplant**: Remove duplicate `Powerplant.set_dispatch_plan()` which broke multi-market bidding
+  - **CSV scenario loader**: Fixed issue when one extra day was being added to the index, which lead to an error in the simulation when additional data was not available in the input data.
+  - **Market opening schedule**: Fixed issue where the market opening was scheduled even though the simulation was ending before the required products. Now the market opening is only scheduled
+    if the total duration of the market products plus first delivery time fits before the simulation end.
+  - **Mango warnings**: Set MANGO logging level to ERROR to avoid unnecessary warnings in the logs.
 
 v0.4.3 - (11th November 2024)
 ===========================================

diff --git a/tests/test_clearing_paper_examples.py b/tests/test_clearing_paper_examples.py
@@ -2,11 +2,11 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+import copy
 import math
 from datetime import datetime, timedelta
 
 from dateutil import rrule as rr
-from dateutil.relativedelta import relativedelta as rd
 
 from assume.common.market_objects import MarketConfig, MarketProduct, Order
 from assume.common.utils import get_available_products
@@ -16,11 +16,12 @@
 
 simple_dayahead_auction_config = MarketConfig(
     market_id="simple_dayahead_auction",
-    market_products=[MarketProduct(rd(hours=+1), 1, rd(hours=1))],
+    market_products=[MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))],
     additional_fields=["node"],
     opening_hours=rr.rrule(
         rr.HOURLY,
         dtstart=datetime(2005, 6, 1),
+        until=datetime(2005, 6, 2),
         cache=True,
     ),
     opening_duration=timedelta(hours=1),
@@ -39,16 +40,15 @@ def test_complex_clearing_whitepaper_a():
     2021
     See Figure 5 a)
     """
-
-    import copy
-
     market_config = copy.copy(simple_dayahead_auction_config)
 
-    market_config.market_products = [MarketProduct(rd(hours=+1), 1, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 1
 
@@ -94,15 +94,15 @@ def test_complex_clearing_whitepaper_d():
     See figure 5 d)
     """
 
-    import copy
-
     market_config = copy.copy(simple_dayahead_auction_config)
-    market_config.market_products = [MarketProduct(rd(hours=+1), 1, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "min_acceptance_ratio",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 1
 
@@ -150,14 +150,14 @@ def test_clearing_non_convex_1():
     5.1.1
     """
 
-    import copy
-
     market_config = copy.copy(simple_dayahead_auction_config)
-    market_config.market_products = [MarketProduct(rd(hours=+1), 3, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 3, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 3
 
@@ -261,15 +261,16 @@ def test_clearing_non_convex_2():
     including mar, BB for gen7
     no load costs cannot be integrated here, so the results differ
     """
-    import copy
 
     market_config = copy.copy(simple_dayahead_auction_config)
-    market_config.market_products = [MarketProduct(rd(hours=+1), 3, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 3, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "min_acceptance_ratio",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 3
 
@@ -379,15 +380,16 @@ def test_clearing_non_convex_3():
 
     half of the demand bids are elastic
     """
-    import copy
 
     market_config = copy.copy(simple_dayahead_auction_config)
-    market_config.market_products = [MarketProduct(rd(hours=+1), 3, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 3, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "min_acceptance_ratio",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 3
 

diff --git a/tests/test_complex_market_mechanisms.py b/tests/test_complex_market_mechanisms.py
@@ -7,7 +7,6 @@
 
 import pandas as pd
 from dateutil import rrule as rr
-from dateutil.relativedelta import relativedelta as rd
 
 from assume.common.market_objects import MarketConfig, MarketProduct
 from assume.common.utils import get_available_products
@@ -17,11 +16,12 @@
 
 simple_dayahead_auction_config = MarketConfig(
     market_id="simple_dayahead_auction",
-    market_products=[MarketProduct(rd(hours=+1), 1, rd(hours=1))],
+    market_products=[MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))],
     additional_fields=["node"],
     opening_hours=rr.rrule(
         rr.HOURLY,
         dtstart=datetime(2005, 6, 1),
+        until=datetime(2005, 6, 2),
         cache=True,
     ),
     opening_duration=timedelta(hours=1),
@@ -37,11 +37,13 @@
 def test_complex_clearing():
     market_config = simple_dayahead_auction_config
     h = 24
-    market_config.market_products = [MarketProduct(rd(hours=+1), h, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), h, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == h
 
@@ -74,7 +76,9 @@ def test_complex_clearing():
 def test_market_coupling():
     market_config = simple_dayahead_auction_config
     h = 2
-    market_config.market_products = [MarketProduct(rd(hours=+1), h, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), h, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "node_id",
@@ -102,7 +106,7 @@ def test_market_coupling():
     grid_data = {"buses": nodes, "lines": lines}
     market_config.param_dict["grid_data"] = grid_data
 
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == h
 
@@ -156,7 +160,9 @@ def test_market_coupling():
 def test_market_coupling_with_island():
     market_config = simple_dayahead_auction_config
     h = 2
-    market_config.market_products = [MarketProduct(rd(hours=+1), h, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), h, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "node_id",
@@ -185,7 +191,7 @@ def test_market_coupling_with_island():
     grid_data = {"buses": nodes, "lines": lines}
     market_config.param_dict["grid_data"] = grid_data
 
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == h
 
@@ -286,12 +292,14 @@ def test_market_coupling_with_island():
 
 def test_complex_clearing_BB():
     market_config = simple_dayahead_auction_config
-    market_config.market_products = [MarketProduct(rd(hours=+1), 2, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 2, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "min_acceptance_ratio",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 2
 
@@ -427,13 +435,15 @@ def test_complex_clearing_BB():
 
 def test_complex_clearing_LB():
     market_config = simple_dayahead_auction_config
-    market_config.market_products = [MarketProduct(rd(hours=+1), 2, rd(hours=1))]
+    market_config.market_products = [
+        MarketProduct(timedelta(hours=1), 2, timedelta(hours=1))
+    ]
     market_config.additional_fields = [
         "bid_type",
         "min_acceptance_ratio",
         "parent_bid_id",
     ]
-    next_opening = market_config.opening_hours.after(datetime.now())
+    next_opening = market_config.opening_hours.after(datetime(2005, 6, 1))
     products = get_available_products(market_config.market_products, next_opening)
     assert len(products) == 2