chore: finalise 3.0.0

Informatievlaanderen · May 23, 2024 · d6c0aea · d6c0aea
2 parents 31fb33b + 9e061fd
commit d6c0aea
Show file tree

Hide file tree

Showing 427 changed files with 5,595 additions and 8,287 deletions.
diff --git a/.gitignore b/.gitignore
@@ -257,3 +257,4 @@ Databases
 
 .sass-cache
 /docs/_site/
+/.jpb/
diff --git a/Dockerfile b/Dockerfile
@@ -17,13 +17,12 @@ FROM amazoncorretto:21-alpine-jdk
 
 COPY --from=app-stage ldes-server-application/target/ldes-server-application.jar ./
 
-COPY --from=app-stage ldes-server-infra-mongo/mongo-repository/target/mongo-repository-jar-with-dependencies.jar ./lib/
-COPY --from=app-stage ldes-server-infra-mongo/mongock-changesets/target/mongock-changesets-jar-with-dependencies.jar ./lib/
-COPY --from=app-stage ldes-server-infra-mongo/mongo-ingest-repository/target/mongo-ingest-repository-jar-with-dependencies.jar ./lib/
-COPY --from=app-stage ldes-server-infra-mongo/mongo-fetch-repository/target/mongo-fetch-repository-jar-with-dependencies.jar ./lib/
-COPY --from=app-stage ldes-server-infra-mongo/mongo-retention-repository/target/mongo-retention-repository-jar-with-dependencies.jar ./lib/
-COPY --from=app-stage ldes-server-infra-mongo/mongo-fragmentation-repository/target/mongo-fragmentation-repository-jar-with-dependencies.jar ./lib/
-COPY --from=app-stage ldes-server-infra-mongo/mongo-admin-repository/target/mongo-admin-repository-jar-with-dependencies.jar ./lib/
+COPY --from=app-stage ldes-server-infra-postgres/postgres-ingest-repository/target/postgres-ingest-repository-jar-with-dependencies.jar ./lib/
+COPY --from=app-stage ldes-server-infra-postgres/postgres-fetch-repository/target/postgres-fetch-repository-jar-with-dependencies.jar ./lib/
+COPY --from=app-stage ldes-server-infra-postgres/postgres-retention-repository/target/postgres-retention-repository-jar-with-dependencies.jar ./lib/
+COPY --from=app-stage ldes-server-infra-postgres/postgres-fragmentation-repository/target/postgres-fragmentation-repository-jar-with-dependencies.jar ./lib/
+COPY --from=app-stage ldes-server-infra-postgres/postgres-admin-repository/target/postgres-admin-repository-jar-with-dependencies.jar ./lib/
+COPY --from=app-stage ldes-server-infra-migration/target/ldes-server-infra-migration-jar-with-dependencies.jar ./lib/
 
 COPY --from=app-stage ldes-server-port-ingest-rest/target/ldes-server-port-ingest-rest-jar-with-dependencies.jar ./lib/
 COPY --from=app-stage ldes-server-port-ingest/target/ldes-server-port-ingest-jar-with-dependencies.jar ./lib/

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -17,16 +17,18 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
-  ldes-mongodb:
-    container_name: ldes-mongodb
-    build:
-      context: ./docker-compose/mongo
+      - postgres
+  postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
     ports:
-      - 27017:27017
+      - 5432:5432
+    environment:
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
     networks:
       - ldes
-    command: --replSet rs0
   pyroscope:
     image: grafana/pyroscope:latest
     ports:

diff --git a/docker-compose/server.config.yml b/docker-compose/server.config.yml
@@ -22,12 +22,14 @@ pyroscope:
   agent:
     enabled: true
 spring:
-  data:
-    mongodb:
-      database: ldes
-      host: ldes-mongodb
-      port: 27017
-      auto-index-creation: true
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
+    database: postgresql
 #server:
 #  servlet:
 #    context-path: /ldes
diff --git a/docs/_configuration/event-stream.md b/docs/_configuration/event-stream.md
@@ -20,6 +20,8 @@ An Event Stream config needs to contain a couple of items:
     * `ldes:createVersions` object that defines whether the LDES should create version objects, indicating the LDES can
       ingest state objects.
       The default value of this object is `false` and the property can be omitted.
+    * `ldes:eventSource` object that defines which members are to be retained in the event stream.
+      When omitted, all members are retained. More info on this can be found [here](./event-stream#configuring-the-member-deletion-on-a-ldes-stream)
 
     * For more info, visit the [Swagger API documentation.](./admin-api)
 
@@ -103,3 +105,37 @@ For more info, visit the [Swagger API documentation.](./admin-api)
     sh:path <https://w3id.org/ldes#versionOfPath>
   ] .
 ````
+
+## Configuring the member deletion on a ldes stream
+
+To determine which members should be permanently deleted from the server, it is necessary to set an event source on each event stream.
+In this object, the retention policies for the event stream can be set. Each member that falls outside of each retention policy will be removed from the event source.
+These members will then be deleted, only if they not a part of any view.
+More information on which retention policies can be used can be found [here](./retention-policies/index)
+
+The retention policies of an event source are the only part of an event stream that can be updated once the event stream is created.
+To do this, there is an endpoint on the admin api. More info can be found [here](./admin-api)
+
+Older versions of the server deleted members the moment they were no longer part of any view, the event source prevents this.
+If this behaviour is wanted, it can be recreated by setting a retention policy which all members will fail.
+Example:
+````turtle
+@prefix ldes: <https://w3id.org/ldes#> .
+
+ldes:retentionPolicy [
+        a ldes:LatestVersionSubset ;
+        ldes:amount 0 ;
+      ] ;
+````
+
+### Example
+
+````turtle
+@prefix ldes: <https://w3id.org/ldes#> .
+
+<> a ldes:EventSource ;
+    ldes:retentionPolicy [
+        a ldes:DurationAgoPolicy ;
+        tree:value "PT5S"^^<http://www.w3.org/2001/XMLSchema#duration> ;
+      ] .
+````
diff --git a/docs/_configuration/openapi.json b/docs/_configuration/openapi.json
@@ -3,7 +3,7 @@
   "info": {
     "title": "LDES Server Admin API",
     "description": "This API makes it possible to manage an LDES Server",
-    "version": "2.15.0-SNAPSHOT"
+    "version": "2.16.0-SNAPSHOT"
   },
   "servers": [
     {

diff --git a/docs/_configuration/retention-policies/index.md b/docs/_configuration/retention-policies/index.md
@@ -11,6 +11,11 @@ has_toc: true
 To reduce storage fill up, it is possible to set a retention policy per view.
 A retention policy has to be added together with its view.
 
+# Deletion Policies
+
+When a member is removed from its last view, they are no longer automatically removed from the event stream.
+
+
 ## Retention polling interval
 By default, every day, the server checks if there are members that can be deleted that are not conform to the retention policy anymore.
 If a higher retention accuracy is desired, or a lower one if resources are limited for example, then a respectively lower or higher retention polling interval can be set via a cron expression.

diff --git a/docs/how-to-run.md b/docs/how-to-run.md
@@ -34,12 +34,13 @@ springdoc:
 ldes-server:
   host-name: "http://localhost:8080"
 spring:
-  data:
-    mongodb:
-      host: ldes-mongodb
-      port: 27017
-      database: ldes
-      auto-index-creation: true
+  datasource:
+    url: jdbc:postgresql://localhost:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
 rest:
   max-age: 120
   max-age-immutable: 604800
@@ -97,36 +98,22 @@ Here is an explanation provided for all the possibilities on how to tweak and co
     <td>No</td>
     <td>604800</td>
   </tr>
-  <tr><td colspan="4"><b>MongoDB Storage</b><sup>2</sup></td></tr>
-  <tr>
-    <td>spring.data.mongodb.host</td>
-    <td>URL that points to the MongoDB server</td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>spring.data.mongodb.port</td>
-    <td>Port on which the MongoDB server runs</td>
-    <td></td>
-    <td></td>
-  </tr>
+  <tr><td colspan="4"><b>PostgreSQL Storage</b><sup>2</sup></td></tr>
   <tr>
-    <td>spring.data.mongodb.database</td>
-    <td>Name for the existing or to be created database on the MongoDB server</td>
+    <td>spring.datasource.url</td>
+    <td>URL that points to the PostgreSQL server</td>
     <td></td>
     <td></td>
   </tr>
   <tr>
-    <td>spring.data.mongodb.uri</td>
-    <td>
-      Alternative to the previous 3 properties, allows passing the mongodb connection string. Note that when a MongoDB link needs to be configured with authentication, it is typically done with an uri, e.g. <code>mongodb://myDatabaseUser:D1fficultP%[email protected]:27017/?authSource=admin</code>
-    </td>
+    <td>spring.datasource.username</td>
+    <td>Username to log into provided PostgreSQL instance</td>
     <td></td>
     <td></td>
   </tr>
   <tr>
-    <td>spring.data.mongodb.auto-index-creation</td>
-    <td>Enables the server to automatically create indices in mongodb. If this property is not enabled, you have to manage the indices manually. This can have a significant impact on performance. <br> We highly advise you to keep this on for performance reasons</td>
+    <td>spring.datasource.password</td>
+    <td>Password to log into provided PostgreSQL instance</td>
     <td></td>
     <td></td>
   </tr>
@@ -180,8 +167,8 @@ Here is an explanation provided for all the possibilities on how to tweak and co
 
 > **Note** 1: The specified url will be prefixed by an optional `server.servlet.context-path`
 
-> **Note** 2: As of this moment the LDES Server only supports a MongoDB implementation. The following properties have
-> to be set to provide connectivity between the server and the database
+> **Note** 2: Since the 3.0 release, the MongoDB got replaced with a PostgreSQL implementation.
+> For Migration instructions, please check the below migration paragraph.
 
 
 > **Note** 3: Unix usually supports a cron expression of 5 parameters, which excludes seconds. However, the spring
@@ -208,17 +195,38 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
-  ldes-mongodb:
-    container_name: ldes-mongodb
-    image: mongo
+      - ldes-postgres
+  postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
     ports:
-      - 27017:27017
+      - 5432:5432
+    environment:
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
     networks:
       - ldes
 networks:
   ldes:
     name: quick_start_network
 ````
 
+## Migration to 3.0
+
+Since the Mongodb implementation got replaced with a PostgreSQL one, a migration path has been provided.
+To enable this, add the following properties to your config:
+
+````yaml
+ldes-server:
+  migrate-mongo: true
+spring:
+  data:
+    mongodb:
+      uri: # connection string pointing to mongodb instance
+  batch:
+    jdbc:
+      initialize-schema: always
+````
+
 [spring documentation]: https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html
diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldes-fragmentisers</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.vsds</groupId>
-        <version>2.15.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/...anderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutor.java b/...anderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutor.java
@@ -51,6 +51,7 @@ private Runnable addMembersToFragments() {
 			var nextMemberToFragment = getNextMemberToFragment(determineLastProcessedSequence());
 
 			while (nextMemberToFragment.isPresent() && isExecutorActive) {
+
 				final FragmentSequence lastProcessedSequence = fragment(nextMemberToFragment.get());
 				nextMemberToFragment = getNextMemberToFragment(lastProcessedSequence);
 			}
@@ -65,7 +66,7 @@ private FragmentSequence determineLastProcessedSequence() {
 	private Optional<Member> getNextMemberToFragment(FragmentSequence lastProcessedSequence) {
 		final String collectionName = viewName.getCollectionName();
 		final long lastProcessedSequenceNr = lastProcessedSequence.sequenceNr();
-		return memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThan(collectionName, lastProcessedSequenceNr);
+		return memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(collectionName, lastProcessedSequenceNr);
 	}
 
 	private FragmentSequence fragment(Member member) {

diff --git a/...e/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java b/...e/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java
@@ -6,5 +6,5 @@
 
 public interface MemberRetriever {
 
-    Optional<Member> findFirstByCollectionNameAndSequenceNrGreaterThan(String collectionName, long sequenceNr);
+    Optional<Member> findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(String collectionName, long sequenceNr);
 }
diff --git a/...aanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetrieverImpl.java b/...aanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetrieverImpl.java
@@ -4,27 +4,27 @@
 import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member;
 import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services.membermapper.MemberMapper;
 import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services.membermapper.MemberMapperCollection;
-import be.vlaanderen.informatievlaanderen.ldes.server.ingest.EventSourceService;
+import be.vlaanderen.informatievlaanderen.ldes.server.ingest.IngestEventSourceService;
 import org.springframework.stereotype.Service;
 
 import java.util.Optional;
 
 @Service
 public class MemberRetrieverImpl implements MemberRetriever {
-    private final EventSourceService eventSourceService;
+    private final IngestEventSourceService eventSourceService;
     private final MemberMapperCollection memberMapperCollection;
 
-    public MemberRetrieverImpl(EventSourceService eventSourceService, MemberMapperCollection memberMapperCollection) {
+    public MemberRetrieverImpl(IngestEventSourceService eventSourceService, MemberMapperCollection memberMapperCollection) {
         this.eventSourceService = eventSourceService;
         this.memberMapperCollection = memberMapperCollection;
     }
 
     @Override
-    public Optional<Member> findFirstByCollectionNameAndSequenceNrGreaterThan(String collectionName, long sequenceNr) {
+    public Optional<Member> findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(String collectionName, long sequenceNr) {
         final MemberMapper memberMapper = memberMapperCollection.getMemberMapper(collectionName)
                 .orElseThrow(() -> new MissingResourceException("eventstream", collectionName));
         return eventSourceService
-                .findFirstByCollectionNameAndSequenceNrGreaterThan(collectionName, sequenceNr)
+                .findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(collectionName, sequenceNr)
                 .map(memberMapper::mapToFragmentationMember);
     }
 }
diff --git a/...ren/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutorTest.java b/...ren/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutorTest.java
@@ -77,7 +77,7 @@ void when_ExecuteIsCalled_then_TheExecutorService_should_NotFragmentAnythingIfNo
 
 			executor.execute();
 
-			verify(memberRetriever).findFirstByCollectionNameAndSequenceNrGreaterThan(viewName.getCollectionName(),
+			verify(memberRetriever).findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(viewName.getCollectionName(),
 					fragmentSequence.sequenceNr());
 			verifyNoMoreInteractions(fragmentationStrategy, rootFragmentRetriever, memberRetriever,
 					fragmentSequenceRepository);
@@ -93,7 +93,7 @@ void when_ExecuteIsCalled_then_TheExecutorService_should_FragmentTheNextMemberIf
 			String memberId = "id";
 			Model memberModel = ModelFactory.createDefaultModel();
 			long sequenceNr = 1L;
-			when(memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThan(viewName.getCollectionName(),
+			when(memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(viewName.getCollectionName(),
 					FragmentSequence.createNeverProcessedSequence(viewName).sequenceNr()))
 					.thenReturn(Optional.of(new Member(memberId, memberModel, sequenceNr)));
 			final Fragment rootFragment = new Fragment(new LdesFragmentIdentifier(viewName, List.of()));

diff --git a/...eren/informatievlaanderen/ldes/server/fragmentation/services/MemberRetrieverImplTest.java b/...eren/informatievlaanderen/ldes/server/fragmentation/services/MemberRetrieverImplTest.java
@@ -3,7 +3,7 @@
 import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException;
 import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services.membermapper.MemberMapper;
 import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services.membermapper.MemberMapperCollection;
-import be.vlaanderen.informatievlaanderen.ldes.server.ingest.EventSourceService;
+import be.vlaanderen.informatievlaanderen.ldes.server.ingest.IngestEventSourceService;
 import be.vlaanderen.informatievlaanderen.ldes.server.ingest.entities.Member;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.junit.jupiter.api.Test;
@@ -28,7 +28,7 @@ class MemberRetrieverImplTest {
     private static final long LAST_PROCESSED_SEQUENCE_NR = 65L;
 
     @Mock
-    private EventSourceService eventSourceService;
+    private IngestEventSourceService eventSourceService;
     @Mock
     private MemberMapperCollection memberMapperCollection;
     @InjectMocks
@@ -37,11 +37,11 @@ class MemberRetrieverImplTest {
     @Test
     void given_MemberMapper_and_IngestMember_when_FindFirst_then_Return_FragmentMember() {
         final MemberMapper memberMapper = new MemberMapper("http://purl.org/dc/terms/isVersionOf", "http://www.w3.org/ns/prov#generatedAtTime");
-        when(eventSourceService.findFirstByCollectionNameAndSequenceNrGreaterThan(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR))
-                .thenReturn(Optional.of(new Member(MEMBER_ID, COLLECTION_NAME, VERSION_OF, TIMESTAMP, LAST_PROCESSED_SEQUENCE_NR + 1, "txId", ModelFactory.createDefaultModel())));
+        when(eventSourceService.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR))
+                .thenReturn(Optional.of(new Member(MEMBER_ID, COLLECTION_NAME, VERSION_OF, TIMESTAMP, LAST_PROCESSED_SEQUENCE_NR + 1, true, "txId", ModelFactory.createDefaultModel())));
         when(memberMapperCollection.getMemberMapper(COLLECTION_NAME)).thenReturn(Optional.of(memberMapper));
 
-        var retrievedMember = memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThan(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR);
+        var retrievedMember = memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR);
 
         assertThat(retrievedMember)
                 .hasValueSatisfying(member -> {
@@ -53,7 +53,7 @@ void given_MemberMapper_and_IngestMember_when_FindFirst_then_Return_FragmentMemb
 
     @Test
     void given_NoMemberMapper_when_FindFirst_then_ThrowException() {
-        assertThatThrownBy(() -> memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThan(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR))
+        assertThatThrownBy(() -> memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR))
                 .isInstanceOf(MissingResourceException.class);
     }
 
@@ -62,7 +62,7 @@ void given_NoIngestMember_when_FindFirst_then_ReturnEmptyOptional() {
         final MemberMapper memberMapper = new MemberMapper("http://purl.org/dc/terms/isVersionOf", "http://www.w3.org/ns/prov#generatedAtTime");
         when(memberMapperCollection.getMemberMapper(COLLECTION_NAME)).thenReturn(Optional.of(memberMapper));
 
-        var retrievedMember = memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThan(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR);
+        var retrievedMember = memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(COLLECTION_NAME, LAST_PROCESSED_SEQUENCE_NR);
 
         assertThat(retrievedMember).isEmpty();
     }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -257,3 +257,4 @@ Databases

		.sass-cache
		/docs/_site/
		/.jpb/