Skip to content

Commit

Permalink
ENTSO-E: fix parsing of XML, resolution fallback, improve error handl…
Browse files Browse the repository at this point in the history 
…ing (#2839)

- automatically choose smallest resolution of available data
- handle exceptions internally
- handle missing values and multiple periods
- add additional tests for real XML
  • Loading branch information
@Sn0w3y
Sn0w3y authored Oct 25, 2024
1 parent d0ae040 commit 7ef6c74
Show file tree
Hide file tree
Showing 4 changed files with 371 additions and 89 deletions.
27 changes: 27 additions & 0 deletions ...ge.timeofusetariff.api/src/io/openems/edge/timeofusetariff/api/utils/ExchangeRateApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.io.IOException;

import io.openems.common.exceptions.OpenemsError.OpenemsNamedException;
import io.openems.common.exceptions.OpenemsException;
import io.openems.edge.common.currency.Currency;
import okhttp3.OkHttpClient;
import okhttp3.Request;
Expand All @@ -28,6 +29,24 @@ public class ExchangeRateApi {

private static final OkHttpClient client = new OkHttpClient();

/**
* Fetches the exchange rate from exchangerate.host.
*
* @param accessKey personal API access key.
* @param source the source currency (e.g. EUR)
* @param target the target currency (e.g. SEK)
* @param orElse the default value
* @return the exchange rate.
*/
public static double getExchangeRateOrElse(String accessKey, String source, Currency target, double orElse) {
try {
return getExchangeRate(accessKey, source, target);
} catch (Exception e) {
e.printStackTrace();
return orElse;
}
}

/**
* Fetches the exchange rate from exchangerate.host.
*
Expand All @@ -40,6 +59,14 @@ public class ExchangeRateApi {
*/
public static double getExchangeRate(String accessKey, String source, Currency target)
throws IOException, OpenemsNamedException {
if (target == Currency.UNDEFINED) {
throw new OpenemsException("Global Currency is UNDEFINED. Please configure it in Core.Meta component");
}

if (target.name().equals(source)) {
return 1.; // No need to fetch exchange rate from API
}

var request = new Request.Builder() //
.url(String.format(BASE_URL, accessKey, source, target.name())) //
.build();
Expand Down
17 changes: 5 additions & 12 deletions ...edge.timeofusetariff.entsoe/src/io/openems/edge/timeofusetariff/entsoe/TouEntsoeImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.openems.edge.timeofusetariff.entsoe;

import static io.openems.common.utils.StringUtils.definedOrElse;
import static io.openems.edge.timeofusetariff.api.utils.ExchangeRateApi.getExchangeRate;
import static io.openems.edge.timeofusetariff.api.utils.ExchangeRateApi.getExchangeRateOrElse;
import static io.openems.edge.timeofusetariff.api.utils.TimeOfUseTariffUtils.generateDebugLog;
import static io.openems.edge.timeofusetariff.entsoe.Utils.parseCurrency;
import static io.openems.edge.timeofusetariff.entsoe.Utils.parsePrices;
Expand Down Expand Up @@ -30,14 +30,11 @@
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

import io.openems.common.exceptions.OpenemsError.OpenemsNamedException;
import io.openems.common.exceptions.OpenemsException;
import io.openems.common.oem.OpenemsEdgeOem;
import io.openems.common.utils.ThreadPoolUtils;
import io.openems.edge.common.channel.value.Value;
import io.openems.edge.common.component.AbstractOpenemsComponent;
import io.openems.edge.common.component.OpenemsComponent;
import io.openems.edge.common.currency.Currency;
import io.openems.edge.common.meta.Meta;
import io.openems.edge.timeofusetariff.api.TimeOfUsePrices;
import io.openems.edge.timeofusetariff.api.TimeOfUseTariff;
Expand Down Expand Up @@ -137,17 +134,13 @@ private synchronized void scheduleTask(long seconds) {
final var result = EntsoeApi.query(token, areaCode, fromDate, toDate);
final var entsoeCurrency = parseCurrency(result);
final var globalCurrency = this.meta.getCurrency();
if (globalCurrency == Currency.UNDEFINED) {
throw new OpenemsException("Global Currency is UNDEFINED. Please configure it in Core.Meta component");
}
final double exchangeRate = getExchangeRateOrElse(//
exchangerateAccesskey, entsoeCurrency, globalCurrency, 1.);

final var exchangeRate = globalCurrency.name().equals(entsoeCurrency) //
? 1. // No need to fetch exchange rate from API.
: getExchangeRate(exchangerateAccesskey, entsoeCurrency, globalCurrency);
// Parse the response for the prices
this.prices.set(parsePrices(result, "PT60M", exchangeRate));
this.prices.set(parsePrices(result, exchangeRate));

} catch (IOException | ParserConfigurationException | SAXException | OpenemsNamedException e) {
} catch (IOException | ParserConfigurationException | SAXException e) {
this.logWarn(this.log, "Unable to Update Entsoe Time-Of-Use Price: " + e.getMessage());
e.printStackTrace();
unableToUpdatePrices = true;
Expand Down
152 changes: 85 additions & 67 deletions io.openems.edge.timeofusetariff.entsoe/src/io/openems/edge/timeofusetariff/entsoe/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,123 +2,141 @@

import static io.openems.common.utils.XmlUtils.stream;
import static java.lang.Double.parseDouble;
import static java.lang.Integer.parseInt;

import java.io.IOException;
import java.io.StringReader;
import java.time.Duration;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeMap;
import java.util.Comparator;
import java.util.function.Function;
import java.util.stream.Stream;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableTable;

import io.openems.common.utils.XmlUtils;
import io.openems.edge.timeofusetariff.api.TimeOfUsePrices;

public class Utils {

private static final DateTimeFormatter FORMATTER_MINUTES = DateTimeFormatter.ofPattern("u-MM-dd'T'HH:mmX");

private static record QueryResult(ZonedDateTime start, List<Float> prices) {
protected static class Builder {
private ZonedDateTime start;
private List<Double> prices = new ArrayList<>();

public Builder start(ZonedDateTime start) {
this.start = start;
return this;
}

public Builder prices(List<Double> prices) {
this.prices.addAll(prices);
return this;
}

public TimeOfUsePrices toTimeOfUsePrices() {
var result = new TreeMap<ZonedDateTime, Double>();
var timestamp = this.start.withZoneSameInstant(ZoneId.systemDefault());
var quarterHourIncrements = this.prices.size() * 4;

for (int i = 0; i < quarterHourIncrements; i++) {
result.put(timestamp, this.prices.get(i / 4));
timestamp = timestamp.plusMinutes(15);
}
return TimeOfUsePrices.from(result);
}
}

public static Builder create() {
return new Builder();
}
}
protected static final DateTimeFormatter FORMATTER_MINUTES = DateTimeFormatter.ofPattern("u-MM-dd'T'HH:mmX");

/**
* Parses the XML response from the Entso-E API to get the Day-Ahead prices.
*
* @param xml The XML string to be parsed.
* @param resolution PT15M or PT60M
* @param exchangeRate The exchange rate of user currency to EUR.
* @return The {@link TimeOfUsePrices}
* @throws ParserConfigurationException on error.
* @throws SAXException on error
* @throws IOException on error
*/
protected static TimeOfUsePrices parsePrices(String xml, String resolution, double exchangeRate)
protected static TimeOfUsePrices parsePrices(String xml, double exchangeRate)
throws ParserConfigurationException, SAXException, IOException {
var root = getXmlRootDocument(xml);

var allPrices = parseXml(root, exchangeRate);

if (allPrices.isEmpty()) {
return TimeOfUsePrices.EMPTY_PRICES;
}

var shortestDuration = allPrices.rowKeySet().stream() //
.sorted() //
.findFirst().get();

final var prices = ImmutableSortedMap.copyOf(allPrices.row(shortestDuration));
final var minTimestamp = prices.firstKey();
final var maxTimestamp = prices.lastKey().plus(shortestDuration);

var result = Stream //
.iterate(minTimestamp, //
t -> t.isBefore(maxTimestamp), //
t -> t.plusMinutes(15)) //
.collect(ImmutableSortedMap.<ZonedDateTime, ZonedDateTime, Double>toImmutableSortedMap(//
Comparator.naturalOrder(), //
Function.identity(), //
t -> prices.floorEntry(t).getValue()));

return TimeOfUsePrices.from(result);
}

protected static Element getXmlRootDocument(String xml)
throws ParserConfigurationException, SAXException, IOException {
var dbFactory = DocumentBuilderFactory.newInstance();
var dBuilder = dbFactory.newDocumentBuilder();
var is = new InputSource(new StringReader(xml));
var doc = dBuilder.parse(is);
var root = doc.getDocumentElement();
var result = QueryResult.create();
return doc.getDocumentElement();
}

protected static ImmutableTable<Duration, ZonedDateTime, Double> parseXml(Element root, double exchangeRate) {
var result = ImmutableTable.<Duration, ZonedDateTime, Double>builder();
stream(root) //
// <TimeSeries>
.filter(n -> n.getNodeName() == "TimeSeries") //
.flatMap(XmlUtils::stream) //
// <Period>
.filter(n -> n.getNodeName() == "Period") //
// Find Period with correct resolution
.filter(p -> stream(p) //
.filter(n -> n.getNodeName() == "resolution") //
.map(XmlUtils::getContentAsString) //
.anyMatch(r -> r.equals(resolution))) //
.forEach(period -> {

var start = ZonedDateTime.parse(//
stream(period) //
// <timeInterval>
.filter(n -> n.getNodeName() == "timeInterval") //
.flatMap(XmlUtils::stream) //
// <start>
.filter(n -> n.getNodeName() == "start") //
.map(XmlUtils::getContentAsString) //
.findFirst().get(),
FORMATTER_MINUTES).withZoneSameInstant(ZoneId.of("UTC"));

if (result.start == null) {
// Avoiding overwriting of start due to multiple periods.
result.start(start);
try {
parsePeriod(result, period, exchangeRate);
} catch (Exception e) {
e.printStackTrace();
}
});
return result.build();
}

result.prices(stream(period) //
// <Point>
.filter(n -> n.getNodeName() == "Point") //
.flatMap(XmlUtils::stream) //
protected static void parsePeriod(ImmutableTable.Builder<Duration, ZonedDateTime, Double> result, Node period,
double exchangeRate) throws Exception {
final var duration = Duration.parse(stream(period) //
// <resolution>
.filter(n -> n.getNodeName() == "resolution") //
.map(XmlUtils::getContentAsString) //
.findFirst().get() /* "PT15M" or "PT60M" */);
final var start = ZonedDateTime.parse(//
stream(period) //
// <timeInterval>
.filter(n -> n.getNodeName() == "timeInterval") //
.flatMap(XmlUtils::stream) //
// <start>
.filter(n -> n.getNodeName() == "start") //
.map(XmlUtils::getContentAsString) //
.findFirst().get(),
FORMATTER_MINUTES).withZoneSameInstant(ZoneId.of("UTC"));

stream(period) //
// <Point>
.filter(n -> n.getNodeName() == "Point") //
.forEach(point -> {
final var position = stream(point) //
// <position>
.filter(n -> n.getNodeName() == "position") //
.map(XmlUtils::getContentAsString) //
.map(s -> parseInt(s)) //
.findFirst().get();
final var timestamp = start.plusMinutes((position - 1) * duration.toMinutes());
final var price = stream(point) //
// <price.amount>
.filter(n -> n.getNodeName() == "price.amount") //
.map(XmlUtils::getContentAsString) //
.map(s -> parseDouble(s) * exchangeRate) //
.toList());
.findFirst().get();
result.put(duration, timestamp, price);
});

return result.toTimeOfUsePrices();
}

/**
Expand Down
Loading

0 comments on commit 7ef6c74

Please sign in to comment.