diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 5d06a00f..c4b38936 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -227,7 +227,7 @@ def handle_output_message(self, content: dict, meta: MetaDict): content_type = content.get("type") market_id = content.get("market_id") - if not content_data: + if content_data is None or len(content_data) == 0: return if content_type in [ @@ -404,7 +404,9 @@ def convert_flows(self, data: dict[tuple[datetime, str], float]): if isinstance(data, pd.DataFrame): df = data - # if data is dict + # if data is list + elif isinstance(data, list): + df = pd.DataFrame.from_dict(data) elif isinstance(data, dict): # Convert the dictionary to a DataFrame df = pd.DataFrame.from_dict( @@ -557,6 +559,7 @@ def create_line(row): continue df["simulation"] = self.simulation_id df.reset_index() + df.columns = df.columns.str.lower() try: with self.db.begin() as db: diff --git a/assume/markets/base_market.py b/assume/markets/base_market.py index 86ca4c35..c99c047d 100644 --- a/assume/markets/base_market.py +++ b/assume/markets/base_market.py @@ -49,10 +49,17 @@ class MarketMechanism: def __init__(self, marketconfig: MarketConfig): super().__init__() self.marketconfig = marketconfig + # calculate last possible market opening as the difference between the market end + # and the length of the longest product plus the delivery time of the products + self.last_market_opening = marketconfig.opening_hours._until - max( + market_product.duration * market_product.count + + market_product.first_delivery + for market_product in marketconfig.market_products + ) def clear( self, orderbook: Orderbook, market_products: list[MarketProduct] - ) -> tuple[Orderbook, Orderbook, list[dict]]: + ) -> tuple[Orderbook, Orderbook, list[dict], dict[tuple, float]]: """ Clears the market. @@ -263,7 +270,7 @@ async def opening(self): # schedule the next opening too next_opening = self.marketconfig.opening_hours.after(market_open) - if next_opening: + if next_opening <= self.last_market_opening: next_opening_ts = datetime2timestamp(next_opening) self.context.schedule_timestamp_task(self.opening(), next_opening_ts) logger.debug( diff --git a/assume/markets/clearing_algorithms/nodal_pricing.py b/assume/markets/clearing_algorithms/nodal_pricing.py index 34eb69d1..ad6cac51 100644 --- a/assume/markets/clearing_algorithms/nodal_pricing.py +++ b/assume/markets/clearing_algorithms/nodal_pricing.py @@ -97,7 +97,7 @@ def setup(self): def clear( self, orderbook: Orderbook, market_products - ) -> tuple[Orderbook, Orderbook, list[dict]]: + ) -> tuple[Orderbook, Orderbook, list[dict], dict[tuple, float]]: """ Clears the market by running a linear optimal power flow (LOPF) with PyPSA. @@ -109,6 +109,8 @@ def clear( Tuple[Orderbook, Orderbook, List[dict]]: The accepted orderbook, rejected orderbook and market metadata. """ + if len(orderbook) <= 0: + return super().clear(orderbook, market_products) orderbook_df = pd.DataFrame(orderbook) orderbook_df["accepted_volume"] = 0.0 orderbook_df["accepted_price"] = 0.0 diff --git a/assume/markets/clearing_algorithms/redispatch.py b/assume/markets/clearing_algorithms/redispatch.py index 7d096822..07f65ba8 100644 --- a/assume/markets/clearing_algorithms/redispatch.py +++ b/assume/markets/clearing_algorithms/redispatch.py @@ -95,7 +95,7 @@ def setup(self): def clear( self, orderbook: Orderbook, market_products - ) -> tuple[Orderbook, Orderbook, list[dict]]: + ) -> tuple[Orderbook, Orderbook, list[dict], dict[tuple, float]]: """ Performs redispatch to resolve congestion in the electricity market. It first checks for congestion in the network and if it finds any, it performs redispatch to resolve it. @@ -110,6 +110,8 @@ def clear( Tuple[Orderbook, Orderbook, List[dict]]: The accepted orderbook, rejected orderbook and market metadata. """ + if len(orderbook) == 0: + return super().clear(orderbook, market_products) orderbook_df = pd.DataFrame(orderbook) orderbook_df["accepted_volume"] = 0.0 orderbook_df["accepted_price"] = 0.0 diff --git a/assume/scenario/loader_csv.py b/assume/scenario/loader_csv.py index 547d818c..c934058b 100644 --- a/assume/scenario/loader_csv.py +++ b/assume/scenario/loader_csv.py @@ -5,7 +5,7 @@ import copy import logging from collections import defaultdict -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path import dateutil.rrule as rr @@ -439,7 +439,7 @@ def load_config_and_create_forecaster( index = pd.date_range( start=start, - end=end + timedelta(days=1), + end=end, freq=config["time_step"], ) diff --git a/assume/world.py b/assume/world.py index 75b21c8d..cf4601a6 100644 --- a/assume/world.py +++ b/assume/world.py @@ -44,7 +44,7 @@ stdout_handler = logging.StreamHandler(stream=sys.stdout) handlers = [file_handler, stdout_handler] logging.basicConfig(level=logging.INFO, handlers=handlers) -logging.getLogger("mango").setLevel(logging.WARNING) +logging.getLogger("mango").setLevel(logging.ERROR) logger = logging.getLogger(__name__) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index f8ef3002..2f1f4324 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -41,6 +41,10 @@ Upcoming Release - **Tutorials**: General fixes of the tutorials, to align with updated functionalitites of Assume - **Tutorial 07**: Aligned Amiris loader with changes in format in Amiris compare (https://gitlab.com/fame-framework/fame-io/-/issues/203 and https://gitlab.com/fame-framework/fame-io/-/issues/208) - **Powerplant**: Remove duplicate `Powerplant.set_dispatch_plan()` which broke multi-market bidding + - **CSV scenario loader**: Fixed issue when one extra day was being added to the index, which lead to an error in the simulation when additional data was not available in the input data. + - **Market opening schedule**: Fixed issue where the market opening was scheduled even though the simulation was ending before the required products. Now the market opening is only scheduled + if the total duration of the market products plus first delivery time fits before the simulation end. + - **Mango warnings**: Set MANGO logging level to ERROR to avoid unnecessary warnings in the logs. v0.4.3 - (11th November 2024) =========================================== diff --git a/tests/test_clearing_paper_examples.py b/tests/test_clearing_paper_examples.py index c1f72104..0db54d97 100644 --- a/tests/test_clearing_paper_examples.py +++ b/tests/test_clearing_paper_examples.py @@ -2,11 +2,11 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +import copy import math from datetime import datetime, timedelta from dateutil import rrule as rr -from dateutil.relativedelta import relativedelta as rd from assume.common.market_objects import MarketConfig, MarketProduct, Order from assume.common.utils import get_available_products @@ -16,11 +16,12 @@ simple_dayahead_auction_config = MarketConfig( market_id="simple_dayahead_auction", - market_products=[MarketProduct(rd(hours=+1), 1, rd(hours=1))], + market_products=[MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))], additional_fields=["node"], opening_hours=rr.rrule( rr.HOURLY, dtstart=datetime(2005, 6, 1), + until=datetime(2005, 6, 2), cache=True, ), opening_duration=timedelta(hours=1), @@ -39,16 +40,15 @@ def test_complex_clearing_whitepaper_a(): 2021 See Figure 5 a) """ - - import copy - market_config = copy.copy(simple_dayahead_auction_config) - market_config.market_products = [MarketProduct(rd(hours=+1), 1, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 1, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 1 @@ -94,15 +94,15 @@ def test_complex_clearing_whitepaper_d(): See figure 5 d) """ - import copy - market_config = copy.copy(simple_dayahead_auction_config) - market_config.market_products = [MarketProduct(rd(hours=+1), 1, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 1, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "min_acceptance_ratio", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 1 @@ -150,14 +150,14 @@ def test_clearing_non_convex_1(): 5.1.1 """ - import copy - market_config = copy.copy(simple_dayahead_auction_config) - market_config.market_products = [MarketProduct(rd(hours=+1), 3, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 3, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 3 @@ -261,15 +261,16 @@ def test_clearing_non_convex_2(): including mar, BB for gen7 no load costs cannot be integrated here, so the results differ """ - import copy market_config = copy.copy(simple_dayahead_auction_config) - market_config.market_products = [MarketProduct(rd(hours=+1), 3, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 3, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "min_acceptance_ratio", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 3 @@ -379,15 +380,16 @@ def test_clearing_non_convex_3(): half of the demand bids are elastic """ - import copy market_config = copy.copy(simple_dayahead_auction_config) - market_config.market_products = [MarketProduct(rd(hours=+1), 3, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 3, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "min_acceptance_ratio", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 3 diff --git a/tests/test_complex_market_mechanisms.py b/tests/test_complex_market_mechanisms.py index 68655eaf..fdd80db0 100644 --- a/tests/test_complex_market_mechanisms.py +++ b/tests/test_complex_market_mechanisms.py @@ -7,7 +7,6 @@ import pandas as pd from dateutil import rrule as rr -from dateutil.relativedelta import relativedelta as rd from assume.common.market_objects import MarketConfig, MarketProduct from assume.common.utils import get_available_products @@ -17,11 +16,12 @@ simple_dayahead_auction_config = MarketConfig( market_id="simple_dayahead_auction", - market_products=[MarketProduct(rd(hours=+1), 1, rd(hours=1))], + market_products=[MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))], additional_fields=["node"], opening_hours=rr.rrule( rr.HOURLY, dtstart=datetime(2005, 6, 1), + until=datetime(2005, 6, 2), cache=True, ), opening_duration=timedelta(hours=1), @@ -37,11 +37,13 @@ def test_complex_clearing(): market_config = simple_dayahead_auction_config h = 24 - market_config.market_products = [MarketProduct(rd(hours=+1), h, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), h, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == h @@ -74,7 +76,9 @@ def test_complex_clearing(): def test_market_coupling(): market_config = simple_dayahead_auction_config h = 2 - market_config.market_products = [MarketProduct(rd(hours=+1), h, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), h, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "node_id", @@ -102,7 +106,7 @@ def test_market_coupling(): grid_data = {"buses": nodes, "lines": lines} market_config.param_dict["grid_data"] = grid_data - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == h @@ -156,7 +160,9 @@ def test_market_coupling(): def test_market_coupling_with_island(): market_config = simple_dayahead_auction_config h = 2 - market_config.market_products = [MarketProduct(rd(hours=+1), h, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), h, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "node_id", @@ -185,7 +191,7 @@ def test_market_coupling_with_island(): grid_data = {"buses": nodes, "lines": lines} market_config.param_dict["grid_data"] = grid_data - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == h @@ -286,12 +292,14 @@ def test_market_coupling_with_island(): def test_complex_clearing_BB(): market_config = simple_dayahead_auction_config - market_config.market_products = [MarketProduct(rd(hours=+1), 2, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 2, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "min_acceptance_ratio", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 2 @@ -427,13 +435,15 @@ def test_complex_clearing_BB(): def test_complex_clearing_LB(): market_config = simple_dayahead_auction_config - market_config.market_products = [MarketProduct(rd(hours=+1), 2, rd(hours=1))] + market_config.market_products = [ + MarketProduct(timedelta(hours=1), 2, timedelta(hours=1)) + ] market_config.additional_fields = [ "bid_type", "min_acceptance_ratio", "parent_bid_id", ] - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 2 diff --git a/tests/test_dmas_market.py b/tests/test_dmas_market.py index 7947cc80..b5bc8c95 100644 --- a/tests/test_dmas_market.py +++ b/tests/test_dmas_market.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from dateutil import rrule as rr -from dateutil.relativedelta import relativedelta as rd from assume.common.market_objects import MarketConfig, MarketProduct, Orderbook from assume.common.utils import get_available_products @@ -18,11 +17,12 @@ simple_dayahead_auction_config = MarketConfig( market_id="simple_dayahead_auction", - market_products=[MarketProduct(rd(hours=+1), 2, rd(hours=1))], + market_products=[MarketProduct(timedelta(hours=1), 2, timedelta(hours=1))], additional_fields=["exclusive_id", "link", "block_id"], opening_hours=rr.rrule( rr.HOURLY, dtstart=datetime(2005, 6, 1), + until=datetime(2005, 6, 2), cache=True, ), opening_duration=timedelta(hours=1), @@ -34,7 +34,9 @@ def test_dmas_market_init(): mr = ComplexDmasClearingRole(simple_dayahead_auction_config) - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -42,7 +44,9 @@ def test_dmas_market_init(): def test_insufficient_generation(): - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -96,7 +100,9 @@ def test_insufficient_generation(): def test_remaining_generation(): - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -175,7 +181,9 @@ def test_remaining_generation(): def test_link_order(): # test not taking a linked order. - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -244,7 +252,9 @@ def test_link_order(): def test_use_link_order(): # test taking a linked order - use more expensive hour 0 to have cheaper overall dispatch. - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -340,7 +350,9 @@ def test_use_link_order(): def test_use_link_order2(): # test taking a linked order - use more expensive hour 0 to have cheaper overall dispatch. - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -454,7 +466,9 @@ def test_market(): sources = [value(model.source[key]) for key in model.source] [model.use_hourly_ask[(block, hour, agent)].value for block, hour, agent in orders["single_ask"].keys()] """ - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) diff --git a/tests/test_simple_market_mechanisms.py b/tests/test_simple_market_mechanisms.py index 6c9f9457..483eb9c1 100644 --- a/tests/test_simple_market_mechanisms.py +++ b/tests/test_simple_market_mechanisms.py @@ -6,7 +6,6 @@ from datetime import datetime, timedelta from dateutil import rrule as rr -from dateutil.relativedelta import relativedelta as rd from assume.common.market_objects import MarketConfig, MarketProduct from assume.common.utils import get_available_products @@ -16,11 +15,12 @@ simple_dayahead_auction_config = MarketConfig( market_id="simple_dayahead_auction", - market_products=[MarketProduct(rd(hours=+1), 1, rd(hours=1))], + market_products=[MarketProduct(timedelta(hours=1), 1, timedelta(hours=1))], additional_fields=["node"], opening_hours=rr.rrule( rr.HOURLY, dtstart=datetime(2005, 6, 1), + until=datetime(2005, 6, 2), cache=True, ), opening_duration=timedelta(hours=1), @@ -32,7 +32,9 @@ def test_market(): - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -73,7 +75,7 @@ async def test_simple_market_mechanism(): print(name) market_config = copy.copy(simple_dayahead_auction_config) market_config.market_mechanism = name - next_opening = market_config.opening_hours.after(datetime.now()) + next_opening = market_config.opening_hours.after(datetime(2005, 6, 1)) products = get_available_products(market_config.market_products, next_opening) assert len(products) == 1 order = { @@ -96,7 +98,9 @@ async def test_simple_market_mechanism(): def test_market_pay_as_clear(): - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -127,7 +131,9 @@ def test_market_pay_as_clear(): def test_market_pay_as_clears_single_demand(): - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening ) @@ -157,7 +163,9 @@ def test_market_pay_as_clears_single_demand(): def test_market_pay_as_clears_single_demand_more_generation(): - next_opening = simple_dayahead_auction_config.opening_hours.after(datetime.now()) + next_opening = simple_dayahead_auction_config.opening_hours.after( + datetime(2005, 6, 1) + ) products = get_available_products( simple_dayahead_auction_config.market_products, next_opening )