Merge branch 'release/2023.10.0'

build.gradle

@@ -247,42 +247,48 @@ task copyBundleReadmes() {
 		def targetBridge = basePath + "edge/bridge.adoc.d"
 		def targetDeviceService = basePath + "edge/device_service.adoc.d"
 		def targetTimedata = basePath + "edge/timedata.adoc.d"
-
+		def targetBackendService = basePath + "backend/service.adoc.d"
+
 		// initialize target files and directories
-		[targetController, targetScheduler, targetNature, targetBridge, targetDeviceService, targetTimedata].each { target ->
-			delete fileTree(dir: target, include: '**/*.adoc') 
+		[targetController, targetScheduler, targetNature, targetBridge, targetDeviceService, targetTimedata, targetBackendService].each { target ->
+			delete fileTree(dir: target, include: '**/*.adoc')
 			new File(target + "/_include.adoc").write('')
 		}
-	
+
 		subprojects.each { proj ->
 			// in each subproject (= bundle)...
 			proj.file(".").listFiles().each { sourceFile ->
 				// find the 'readme.adoc' file
 				if(sourceFile.getName().equalsIgnoreCase("readme.adoc")) {
 					def bundle = sourceFile.getParentFile().getName()
 					def target = null
-					// evaluate the OpenEMS Component ('Backend' or 'Edge')
+
 					if(bundle.startsWith("io.openems.edge.")) {
-					// evaluate the bundle type (e.g. 'Controller')
-					def edgeBundle = bundle.substring("io.openems.edge.".length())
-					if(edgeBundle.endsWith(".api")) {
-						target = targetNature
-					} else if(edgeBundle.startsWith("controller.")) {
-						target = targetController
-					} else if(edgeBundle.startsWith("scheduler.")) {
-						target = targetScheduler
-					} else if(edgeBundle.startsWith("bridge.")) {
-						target = targetBridge
-					} else if(edgeBundle.startsWith("timedata.")) {
-						target = targetTimedata
-					} else {
-						target = targetDeviceService
-					}
+						def edgeBundle = bundle.substring("io.openems.edge.".length())
+						if(edgeBundle.endsWith(".api")) {
+							target = targetNature
+						} else if(edgeBundle.startsWith("controller.")) {
+							target = targetController
+						} else if(edgeBundle.startsWith("scheduler.")) {
+							target = targetScheduler
+						} else if(edgeBundle.startsWith("bridge.")) {
+							target = targetBridge
+						} else if(edgeBundle.startsWith("timedata.")) {
+							target = targetTimedata
+						} else {
+							target = targetDeviceService
+						}
 
 					} else if(bundle.startsWith("io.openems.backend.")) {
-						// ignore
-						return
-
+						def backendBundle = bundle.substring("io.openems.backend.".length())
+						if(backendBundle.startsWith("timedata.")) {
+							target = targetBackendService
+						} else if(backendBundle.startsWith("metadata.")) {
+							target = targetBackendService
+						} else {
+							return // ignore
+						}
+
 					} else if(bundle.startsWith("io.openems.wrapper")) {
 						// ignore
 						return

cnf/pom.xml

@@ -221,7 +221,7 @@
 			<!-- Changelog: https://github.com/apache/felix-dev/blob/master/inventory/changelog.txt -->
 			<groupId>org.apache.felix</groupId>
 			<artifactId>org.apache.felix.inventory</artifactId>
-			<version>1.1.0</version>
+			<version>2.0.0</version>
 		</dependency>
 		<dependency>
 			<!-- Apache Felix Metatype Service -->
@@ -239,17 +239,17 @@
 		</dependency>
 		<dependency>
 			<!-- Apache Felix Web Management Console -->
-			<!-- Changelog: https://github.com/apache/felix-dev/blob/master/webconsole/changelog.txt -->
+			<!-- Changelog: https://github.com/apache/felix-dev/blob/master/webconsole/README.md#releases -->
 			<groupId>org.apache.felix</groupId>
 			<artifactId>org.apache.felix.webconsole</artifactId>
-			<version>4.7.2</version>
+			<version>4.9.4</version>
 		</dependency>
 		<dependency>
 			<!-- Apache Felix Web Console Service Component Runtime/Declarative Services Plugin -->
 			<!-- Changelog: https://github.com/apache/felix-dev/blob/master/webconsole-plugins/ds/changelog.txt -->
 			<groupId>org.apache.felix</groupId>
 			<artifactId>org.apache.felix.webconsole.plugins.ds</artifactId>
-			<version>2.2.0</version>
+			<version>2.3.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.servicemix.bundles</groupId>
@@ -261,12 +261,12 @@
 			<!-- Changelog: https://github.com/typetools/checker-framework/blob/master/docs/CHANGELOG.md -->
 			<groupId>org.checkerframework</groupId>
 			<artifactId>checker-qual</artifactId>
-			<version>3.37.0</version>
+			<version>3.38.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.dhatim</groupId>
 			<artifactId>fastexcel</artifactId>
-			<version>0.15.6</version>
+			<version>0.15.7</version>
 		</dependency>
 		<dependency>
 			<!-- Eclipse Paho MQTTv5 Client -->
@@ -390,6 +390,12 @@
 			<artifactId>org.osgi.util.promise</artifactId>
 			<version>1.3.0</version>
 		</dependency>
+		<dependency>
+			<!-- Used by Apache Felix Web Management Console -->
+			<groupId>org.owasp.encoder</groupId>
+			<artifactId>encoder</artifactId>
+			<version>1.2.3</version>
+		</dependency>
 		<dependency>
 			<!-- Used by io.openems.backend.metadata.odoo -->
 			<groupId>org.postgresql</groupId>

doc/modules/ROOT/nav.adoc

@@ -21,6 +21,9 @@
 * OpenEMS Backend
 ** xref:backend/architecture.adoc[Architecture]
 ** xref:backend/backend-to-backend.adoc[Backend-to-Backend]
+** xref:backend/metadata.adoc[Metadata]
+** xref:backend/timedata.adoc[Timedata]
+** xref:backend/service.adoc[Service]
 ** xref:backend/build.adoc[Build OpenEMS Backend]
 ** xref:backend/deploy.adoc[Deploy OpenEMS Backend]
 * xref:component-communication/index.adoc[Component Communication]

doc/modules/ROOT/pages/backend/metadata.adoc

@@ -0,0 +1,37 @@
+= Metadata
+:sectnums:
+:sectnumlevels: 4
+:toc:
+:toclevels: 4
+:experimental:
+:keywords: AsciiDoc
+:source-highlighter: highlight.js
+:icons: font
+:imagesdir: ../../assets/images
+
+The https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.common/src/io/openems/backend/common/metadata/Metadata.java[Metadata] interface defines a service, that coordinates OpenEMS Backend communication with one or more OpenEMS Edges and with the OpenEMS UI.
+It is responsible for identification and authorization of OpenEMS edges and of Users using the OpenEMS UI.
+Beside that the Metadata bundle is responsible for the state of https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.common/src/io/openems/backend/common/metadata/User.java[users] and https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.common/src/io/openems/backend/common/metadata/Edge.java[edges] and related processes.
+
+== Edges
+
+The Metadata service provides access to the https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.common/src/io/openems/backend/common/metadata/Edge.java[Edge] object - the digital twin representation of the OpenEMS Edge instance.
+[NOTE]
+====
+Identification of OpenEMS Edges is done with a unique API key.
+Please take care, that the API key includes a lot of randomness, because it is implicitly used to authorize the Edge.
+====
+
+== Users
+
+The https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.common/src/io/openems/backend/common/metadata/User.java[User] object includes meta information about a user. 
+Furthermore it is used for identification/authorization of the user.
+The metadata service can be used to change various attributes which are associated with a user (e.g. the users default language).
+
+== Processes
+
+The metadata service provides methods to
+
+* add users to an Edge,
+* set and change alerting/notification settings,
+* set up new Edge devices.

doc/modules/ROOT/pages/backend/service.adoc

@@ -0,0 +1,18 @@
+= Backend Services
+:sectnums:
+:sectnumlevels: 4
+:toc:
+:toclevels: 4
+:experimental:
+:keywords: AsciiDoc
+:source-highlighter: highlight.js
+:icons: font
+:imagesdir: ../../assets/images
+
+The following bundles are available in OpenEMS Backend.
+[NOTE]
+====
+This list is not complete.
+====
+
+include::service.adoc.d/_include.adoc[leveloffset=+0]

doc/modules/ROOT/pages/backend/service.adoc.d/.gitignore

@@ -0,0 +1 @@
+*.adoc

doc/modules/ROOT/pages/backend/timedata.adoc

@@ -0,0 +1,45 @@
+= Timedata
+:sectnums:
+:sectnumlevels: 4
+:toc:
+:toclevels: 4
+:experimental:
+:keywords: AsciiDoc
+:source-highlighter: highlight.js
+:icons: font
+:imagesdir: ../../assets/images
+
+Live and historical data of an OpenEMS Edge are handled by a https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.common/src/io/openems/backend/common/timedata/Timedata.java[Timedata] service.
+It describes basically methods to write and read Edge data to/from a database. There are different kind of timedata providers within OpenEMS (see xref:service.adoc[Service] for concrete implementations).
+
+Within OpenEMS Backend the only component which uses the different timedata services directly is the https://github.com/OpenEMS/openems/blob/develop/io.openems.backend.core/src/io/openems/backend/core/timedatamanager/TimedataManagerImpl.java[TimedataManager].
+It passes Edge relevant data to **all** Timedata service and it reads the data from the **first** Timedata providers which can deliver it. 
+
+[NOTE]
+====
+Following paragraphs describe the new data handlng since OpenEMS version 2023.8.
+====
+
+OpenEMS Edges will send different types of data to the OpenEMS Backend:
+
+* `TimestampedDataNotification` 
+ ** channel values which have changed 
+ ** every 5 minutes a full snapshot of all channel values is sent, including channels which haven't changed over this time period
+
+* `AggregatedDataNotification`
+ ** data is sent in a format that is optimized for fast querying from the database, to allow very fast responses in OpenEMS UI
+ ** aggregated (average or maximum) channel values are sent every 5 minutes
+ ** Backend services handle values differently according to their aggregation type, e.g. 5-minute-average is always stored; maximum is only stored if feasible for fast energy queries (e.g. at the end of a day, depending on a time-zone)
+
+* `ResendDataNotification`
+ ** historic data that is resent after connection problems between Edge and Backend
+ ** this data is always aggregated in the form of the Edge.Timedata service (e.g. RRD4j) 
+
+Splitting the data enables OpenEMS to implement different timedata providers, 
+which handle data differently. This gives more flexibility when scaling OpenEMS.  
+Also due to performance reasons the computation of `AggregatedDataNotification` is done on the Edge side. 
+This helps keeping CPU load on the database server low.
+
+To get a better understanding of the new Edge data concept, have a look at the   
+xref:service.adoc.d/io.openems.backend.timedata.aggregatedinflux.adoc[Aggregated Influx] bundle.
+

io.openems.backend.alerting/test/io/openems/backend/alerting/handler/TestOfflineEdgeHandler.java

@@ -45,8 +45,8 @@ public TestOfflineEdgeHandler() {
 				));
 	}
 
-	private Edge getTestEdge(Metadata metadata, String id, ZonedDateTime LastMessage, boolean isOnline) {
-		final var edge = new Edge(metadata, id, "comment", "version", "producttype", LastMessage);
+	private Edge getTestEdge(Metadata metadata, String id, ZonedDateTime lastMessage, boolean isOnline) {
+		final var edge = new Edge(metadata, id, "comment", "version", "producttype", lastMessage);
 		edge.setOnline(isOnline);
 		return edge;
 	}

io.openems.backend.application/BackendApp.bndrun

@@ -60,7 +60,7 @@
 
 -runbundles: \
 	Java-WebSocket;version='[1.5.4,1.5.5)',\
-	checker-qual;version='[3.37.0,3.37.1)',\
+	checker-qual;version='[3.38.0,3.38.1)',\
 	com.google.gson;version='[2.10.1,2.10.2)',\
 	com.google.guava;version='[32.1.2,32.1.3)',\
 	com.google.guava.failureaccess;version='[1.0.1,1.0.2)',\
@@ -108,11 +108,11 @@
 	org.apache.felix.fileinstall;version='[3.7.4,3.7.5)',\
 	org.apache.felix.http.jetty;version='[5.1.0,5.1.1)',\
 	org.apache.felix.http.servlet-api;version='[2.1.0,2.1.1)',\
-	org.apache.felix.inventory;version='[1.1.0,1.1.1)',\
+	org.apache.felix.inventory;version='[2.0.0,2.0.1)',\
 	org.apache.felix.metatype;version='[1.2.4,1.2.5)',\
 	org.apache.felix.scr;version='[2.2.6,2.2.7)',\
-	org.apache.felix.webconsole;version='[4.7.2,4.7.3)',\
-	org.apache.felix.webconsole.plugins.ds;version='[2.2.0,2.2.1)',\
+	org.apache.felix.webconsole;version='[4.9.4,4.9.5)',\
+	org.apache.felix.webconsole.plugins.ds;version='[2.3.0,2.3.1)',\
 	org.jetbrains.kotlin.osgi-bundle;version='[1.9.10,1.9.11)',\
 	org.jsr-305;version='[3.0.2,3.0.3)',\
 	org.ops4j.pax.logging.pax-logging-api;version='[2.2.1,2.2.2)',\
@@ -121,5 +121,6 @@
 	org.osgi.service.jdbc;version='[1.1.0,1.1.1)',\
 	org.osgi.util.function;version='[1.2.0,1.2.1)',\
 	org.osgi.util.promise;version='[1.3.0,1.3.1)',\
+	org.owasp.encoder;version='[1.2.3,1.2.4)',\
 	org.postgresql.jdbc;version='[42.6.0,42.6.1)',\
 	reactive-streams;version='[1.0.4,1.0.5)'

io.openems.backend.common/src/io/openems/backend/common/edgewebsocket/EdgeWebsocket.java

@@ -2,6 +2,7 @@
 
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 
 import org.osgi.annotation.versioning.ProviderType;
@@ -45,15 +46,15 @@ public CompletableFuture<JsonrpcResponseSuccess> send(String edgeId, User user,
 	/**
 	 * Handles a {@link SubscribeSystemLogRequest}.
 	 *
-	 * @param edgeId  the Edge-ID
-	 * @param user    the {@link User}
-	 * @param token   the UI session token
-	 * @param request the {@link SubscribeSystemLogRequest}
+	 * @param edgeId      the Edge-ID
+	 * @param user        the {@link User}
+	 * @param websocketId the id of the UI websocket connection
+	 * @param request     the {@link SubscribeSystemLogRequest}
 	 * @return a reply
 	 * @throws OpenemsNamedException on error
 	 */
 	public CompletableFuture<JsonrpcResponseSuccess> handleSubscribeSystemLogRequest(String edgeId, User user,
-			String token, SubscribeSystemLogRequest request) throws OpenemsNamedException;
+			UUID websocketId, SubscribeSystemLogRequest request) throws OpenemsNamedException;
 
 	/**
 	 * Gets the latest values for the given ChannelAddresses.

io.openems.backend.common/src/io/openems/backend/common/metadata/Edge.java

@@ -25,7 +25,7 @@ public class Edge {
 	private String comment;
 	private SemanticVersion version;
 	private String producttype;
-	private ZonedDateTime lastmessage = null;
+	private ZonedDateTime lastmessage;
 	private boolean isOnline = false;
 
 	private final List<EdgeUser> user;

io.openems.backend.common/src/io/openems/backend/common/metadata/Metadata.java

@@ -21,8 +21,8 @@
 import io.openems.common.exceptions.OpenemsError.OpenemsNamedException;
 import io.openems.common.exceptions.OpenemsException;
 import io.openems.common.jsonrpc.request.GetEdgesRequest.PaginationOptions;
+import io.openems.common.jsonrpc.response.GetEdgesResponse.EdgeMetadata;
 import io.openems.common.session.Language;
-import io.openems.common.session.Role;
 import io.openems.common.types.ChannelAddress;
 import io.openems.common.types.EdgeConfig;
 import io.openems.common.types.EdgeConfig.Component.Channel;
@@ -352,7 +352,8 @@ public static final class Events {
 	 * @return the role to the Edge-IDs
 	 * @throws OpenemsNamedException on error
 	 */
-	public Map<String, Role> getPageDevice(User user, PaginationOptions paginationOptions) throws OpenemsNamedException;
+	public List<EdgeMetadata> getPageDevice(User user, PaginationOptions paginationOptions)
+			throws OpenemsNamedException;
 
 	/**
 	 * Gets the Role for a edge of the current user.
@@ -362,6 +363,6 @@ public static final class Events {
 	 * @return the role to the edge
 	 * @throws OpenemsNamedException on error
 	 */
-	public Role getRoleForEdge(User user, String edgeId) throws OpenemsNamedException;
+	public EdgeMetadata getEdgeMetadataForUser(User user, String edgeId) throws OpenemsNamedException;
 
 }

io.openems.backend.common/src/io/openems/backend/common/metadata/User.java

@@ -6,6 +6,7 @@
 import java.util.NavigableMap;
 import java.util.TreeMap;
 
+import io.openems.common.channel.Level;
 import io.openems.common.exceptions.OpenemsError;
 import io.openems.common.exceptions.OpenemsError.OpenemsNamedException;
 import io.openems.common.jsonrpc.response.GetEdgesResponse.EdgeMetadata;
@@ -92,7 +93,9 @@ public static List<EdgeMetadata> generateEdgeMetadatas(User user, Metadata metad
 						edge.getVersion(), // Version
 						role, // Role
 						edge.isOnline(), // Online-State
-						edge.getLastmessage() // Last-Message Timestamp
+						edge.getLastmessage(), // Last-Message Timestamp
+						null, //
+						Level.OK //
 				));
 			}
 		}

io.openems.backend.common/src/io/openems/backend/common/test/DummyMetadata.java

@@ -18,8 +18,8 @@
 import io.openems.common.exceptions.OpenemsError.OpenemsNamedException;
 import io.openems.common.exceptions.OpenemsException;
 import io.openems.common.jsonrpc.request.GetEdgesRequest.PaginationOptions;
+import io.openems.common.jsonrpc.response.GetEdgesResponse.EdgeMetadata;
 import io.openems.common.session.Language;
-import io.openems.common.session.Role;
 
 public class DummyMetadata implements Metadata {
 	@Override
@@ -138,13 +138,13 @@ public Optional<String> getSerialNumberForEdge(Edge edge) {
 	}
 
 	@Override
-	public Map<String, Role> getPageDevice(User user, PaginationOptions paginationOptions)
+	public List<EdgeMetadata> getPageDevice(User user, PaginationOptions paginationOptions)
 			throws OpenemsNamedException {
 		throw new UnsupportedOperationException("Unsupported by Dummy Class");
 	}
 
 	@Override
-	public Role getRoleForEdge(User user, String edgeId) throws OpenemsNamedException {
+	public EdgeMetadata getEdgeMetadataForUser(User user, String edgeId) throws OpenemsNamedException {
 		throw new UnsupportedOperationException("Unsupported by Dummy Class");
 	}