From deb57fbb0753ef144a15156802a7255a8f960e9b Mon Sep 17 00:00:00 2001 From: Yalz Date: Thu, 23 May 2024 16:06:07 +0200 Subject: [PATCH 01/15] chore: prepare 3.1.0 release + delete migration scripts --- .../ldes-fragmentisers-common/pom.xml | 2 +- .../ldes-fragmentisers-geospatial/pom.xml | 2 +- .../ldes-fragmentisers-pagination/pom.xml | 2 +- .../ldes-fragmentisers-reference/pom.xml | 2 +- .../pom.xml | 2 +- ldes-fragmentisers/pom.xml | 2 +- ldes-server-admin/pom.xml | 2 +- ldes-server-application/pom.xml | 2 +- .../ldes/server/Application.java | 10 +- ldes-server-compaction/pom.xml | 2 +- ldes-server-domain/pom.xml | 2 +- ldes-server-infra-migration/pom.xml | 136 ----------- .../ldes/server/MongoConfiguration.java | 29 --- .../ldes/server/MongoToPostgresMigration.java | 43 ---- .../server/admin/AdminMigrationConfig.java | 38 ---- .../dcatdataset/DcatDatasetMigration.java | 56 ----- .../dcatdataset/entity/DcatDatasetEntity.java | 34 --- .../dcatserver/DcatCatalogMigration.java | 56 ----- .../dcatserver/entity/DcatCatalogEntity.java | 33 --- .../eventstream/EventStreamMigration.java | 57 ----- .../eventstream/entity/EventStreamEntity.java | 41 ---- .../mongo/shaclshape/ShaclShapeMigration.java | 57 ----- .../shaclshape/entity/ShaclShapeEntity.java | 33 --- .../mongo/view/DataServiceMigration.java | 57 ----- .../admin/mongo/view/ViewMigration.java | 57 ----- .../mongo/view/entity/DataServiceEntity.java | 37 --- .../admin/mongo/view/entity/ViewEntity.java | 53 ----- .../server/fetch/FetchMigrationConfig.java | 67 ------ .../fetch/entity/MemberAllocationEntity.java | 50 ---- .../FragmentationMigrationConfig.java | 117 ---------- .../fragmentation/entity/FragmentEntity.java | 98 -------- .../fragmentation/entity/SequenceEntity.java | 26 --- .../server/ingest/IngestMigrationConfig.java | 66 ------ .../server/ingest/entity/MemberEntity.java | 86 ------- .../retention/RetentionMigrationConfig.java | 67 ------ .../entities/MemberPropertiesEntity.java | 60 ----- .../ldes/server/AdminMigrationTest.java | 215 ------------------ .../ldes/server/FetchMigrationTest.java | 68 ------ .../ldes/server/FragmentMigrationTest.java | 120 ---------- .../ldes/server/IngestMigrationTest.java | 97 -------- .../ldes/server/MongoTestConfiguration.java | 36 --- .../server/PostgresTestConfiguration.java | 46 ---- .../ldes/server/RetentionMigrationTest.java | 73 ------ .../resources/admin/view/retentionpolicy.nq | 2 - .../test/resources/admin/view/view_valid.ttl | 41 ---- .../src/test/resources/application-test.yml | 24 -- .../src/test/resources/randomModel.nq | 23 -- .../src/test/resources/randomModel.ttl | 22 -- ldes-server-infra-postgres/pom.xml | 2 +- .../postgres-admin-repository/pom.xml | 2 +- .../postgres-fetch-repository/pom.xml | 2 +- .../postgres-fragmentation-repository/pom.xml | 2 +- .../postgres-ingest-repository/pom.xml | 2 +- .../postgres-retention-repository/pom.xml | 2 +- ldes-server-instrumentation/pom.xml | 2 +- ldes-server-integration-test/pom.xml | 2 +- ldes-server-port-fetch-rest/pom.xml | 2 +- ldes-server-port-fetch/pom.xml | 2 +- ldes-server-port-ingest-rest/pom.xml | 2 +- ldes-server-port-ingest/pom.xml | 2 +- ldes-server-retention/pom.xml | 2 +- pom.xml | 3 +- 62 files changed, 25 insertions(+), 2255 deletions(-) delete mode 100644 ldes-server-infra-migration/pom.xml delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoConfiguration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoToPostgresMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/AdminMigrationConfig.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/DcatDatasetMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/entity/DcatDatasetEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/DcatCatalogMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/entity/DcatCatalogEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/EventStreamMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/entity/EventStreamEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/ShaclShapeMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/entity/ShaclShapeEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/DataServiceMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/ViewMigration.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/DataServiceEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/ViewEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/FetchMigrationConfig.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/entity/MemberAllocationEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationMigrationConfig.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/FragmentEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/SequenceEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/IngestMigrationConfig.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/entity/MemberEntity.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/RetentionMigrationConfig.java delete mode 100644 ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberPropertiesEntity.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/AdminMigrationTest.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FetchMigrationTest.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentMigrationTest.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/IngestMigrationTest.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoTestConfiguration.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/PostgresTestConfiguration.java delete mode 100644 ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/RetentionMigrationTest.java delete mode 100644 ldes-server-infra-migration/src/test/resources/admin/view/retentionpolicy.nq delete mode 100644 ldes-server-infra-migration/src/test/resources/admin/view/view_valid.ttl delete mode 100644 ldes-server-infra-migration/src/test/resources/application-test.yml delete mode 100644 ldes-server-infra-migration/src/test/resources/randomModel.nq delete mode 100644 ldes-server-infra-migration/src/test/resources/randomModel.ttl diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml index 997aecaedd..7c54dc22a1 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml index cca0be66e5..2c1037a968 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml @@ -3,7 +3,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml index d0597ff5f2..9c53ee94b2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml @@ -3,7 +3,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml index a505110469..6eea043336 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT 4.0.0 jar diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml index 71a32454b7..0bbb1c6fee 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-fragmentisers - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-fragmentisers-timebased-hierarchical diff --git a/ldes-fragmentisers/pom.xml b/ldes-fragmentisers/pom.xml index 06de70319d..b0cc9ab2bf 100644 --- a/ldes-fragmentisers/pom.xml +++ b/ldes-fragmentisers/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-fragmentisers diff --git a/ldes-server-admin/pom.xml b/ldes-server-admin/pom.xml index 28812897bf..956339c2ab 100644 --- a/ldes-server-admin/pom.xml +++ b/ldes-server-admin/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index 138043a4b5..7142b2b5fb 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-application diff --git a/ldes-server-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/Application.java b/ldes-server-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/Application.java index fa2fe99916..8a20cf7b41 100644 --- a/ldes-server-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/Application.java +++ b/ldes-server-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/Application.java @@ -4,17 +4,9 @@ import io.swagger.v3.oas.annotations.servers.Server; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; -import org.springframework.boot.autoconfigure.data.mongo.MongoRepositoriesAutoConfiguration; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; @OpenAPIDefinition(servers = { @Server(url = "/", description = "Default Server URL") }) -//TODO Cleanup after 3.0 release -@SpringBootApplication(exclude = { - MongoAutoConfiguration.class, - MongoRepositoriesAutoConfiguration.class, - MongoDataAutoConfiguration.class -}) +@SpringBootApplication public class Application { public static void main(String[] args) { SpringApplication.run(Application.class, args); diff --git a/ldes-server-compaction/pom.xml b/ldes-server-compaction/pom.xml index 52c5d7d3f9..54c00eddfb 100644 --- a/ldes-server-compaction/pom.xml +++ b/ldes-server-compaction/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-compaction diff --git a/ldes-server-domain/pom.xml b/ldes-server-domain/pom.xml index 1f45a0c30b..277bd3705c 100644 --- a/ldes-server-domain/pom.xml +++ b/ldes-server-domain/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-domain diff --git a/ldes-server-infra-migration/pom.xml b/ldes-server-infra-migration/pom.xml deleted file mode 100644 index e09e91d4d3..0000000000 --- a/ldes-server-infra-migration/pom.xml +++ /dev/null @@ -1,136 +0,0 @@ - - - 4.0.0 - - be.vlaanderen.informatievlaanderen.vsds - ldes-server - 3.0.0-SNAPSHOT - - - ldes-server-infra-migration - - - 42.6.0 - 3.7.4 - - - - - org.springframework.boot - spring-boot-starter-batch - - - org.springframework.batch - spring-batch-test - test - - - org.springframework.boot - spring-boot-test - test - - - - - org.springframework.boot - spring-boot-starter-data-jpa - - - io.hypersistence - hypersistence-utils-hibernate-63 - ${hibernate-extensions.version} - - - org.postgresql - postgresql - ${postgresql.version} - - - - - org.springframework.data - spring-data-mongodb - 4.1.5 - compile - - - org.mongodb - mongodb-driver-sync - 4.9.1 - compile - - - - - be.vlaanderen.informatievlaanderen.vsds - postgres-fetch-repository - ${project.version} - - - be.vlaanderen.informatievlaanderen.vsds - postgres-ingest-repository - ${project.version} - - - be.vlaanderen.informatievlaanderen.vsds - postgres-fragmentation-repository - ${project.version} - - - be.vlaanderen.informatievlaanderen.vsds - postgres-retention-repository - ${project.version} - - - be.vlaanderen.informatievlaanderen.vsds - postgres-admin-repository - ${project.version} - - - be.vlaanderen.informatievlaanderen.vsds - ldes-fragmentisers-pagination - ${project.version} - test - - - - org.testcontainers - junit-jupiter - 1.18.3 - test - - - org.testcontainers - postgresql - 1.18.3 - test - - - org.testcontainers - mongodb - 1.18.3 - test - - - org.springframework.boot - spring-boot-test-autoconfigure - 3.2.3 - test - - - - - - - ${project.artifactId} - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - \ No newline at end of file diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoConfiguration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoConfiguration.java deleted file mode 100644 index 12a5845a38..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoConfiguration.java +++ /dev/null @@ -1,29 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class MongoConfiguration { - - @Value("${spring.data.mongodb.uri}") - private String mongoUri; - - @Bean - @ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") - public MongoDatabaseFactory mongoDatabaseFactory() { - return new SimpleMongoClientDatabaseFactory(mongoUri); - } - - @Bean - @ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") - public MongoTemplate mongoTemplate(MongoDatabaseFactory mongoDbFactory) { - return new MongoTemplate(mongoDbFactory); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoToPostgresMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoToPostgresMigration.java deleted file mode 100644 index f6fb78146e..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoToPostgresMigration.java +++ /dev/null @@ -1,43 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; - -import java.util.List; - -@Configuration -@Profile("!test") -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class MongoToPostgresMigration { - - private final JobLauncher jobLauncher; - private final List migrationJobs; - - public MongoToPostgresMigration(JobLauncher jobLauncher, - @Qualifier("migrationMongoIngest") Job ingestMigration, - @Qualifier("migrationMongoFetch") Job fetchMigration, - @Qualifier("migrationMongoFragmentation") Job fragmentMigration, - @Qualifier("migrationMongoRetention") Job retentionMigration, - @Qualifier("migrationMongoAdmin") Job adminMigration) { - this.jobLauncher = jobLauncher; - this.migrationJobs = List.of(ingestMigration, fetchMigration, fragmentMigration, - retentionMigration, adminMigration); - } - - @Bean - public CommandLineRunner commandLineRunner() { - return args -> { - for (Job job : migrationJobs) { - jobLauncher.run(job, new JobParameters()); - } - }; - } - -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/AdminMigrationConfig.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/AdminMigrationConfig.java deleted file mode 100644 index 0616c9cc80..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/AdminMigrationConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin; - -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class AdminMigrationConfig { - - @Bean("migrationMongoAdmin") - public Job memberEntityMigrationJob(JobRepository jobRepository, - @Qualifier("migrationDcatDataset") Step dcatDatasetStep, - @Qualifier("migrationDcatCatalog") Step dcatCatalogStep, - @Qualifier("migrationEventStream") Step eventStreamStep, - @Qualifier("migrationShaclShape") Step shaclShapeStep, - @Qualifier("migrationDataService") Step dataServiceStep, - @Qualifier("migrationView") Step viewStep - ) { - - return new JobBuilder("migrationMongoAdmin", jobRepository) - .incrementer(new RunIdIncrementer()) - .flow(dcatDatasetStep) - .next(dcatCatalogStep) - .next(eventStreamStep) - .next(shaclShapeStep) - .next(dataServiceStep) - .next(viewStep) - .end() - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/DcatDatasetMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/DcatDatasetMigration.java deleted file mode 100644 index 58b0d873b4..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/DcatDatasetMigration.java +++ /dev/null @@ -1,56 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatdataset; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatdataset.entity.DcatDatasetEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatdataset.service.DcatDatasetEntityConverter; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class DcatDatasetMigration { - @Bean - public MongoItemReader dcatDatasetEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatdataset.entity.DcatDatasetEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor dcatDatasetEntityProcessor(DcatDatasetEntityConverter mapper) { - return noSQLData -> mapper.datasetToEntity(noSQLData.toDcatDataset()); - } - - @Bean - public JpaItemWriter dcatDatasetEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationDcatDataset") - public Step migrationDcatDataset(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationDcatDataset", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/entity/DcatDatasetEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/entity/DcatDatasetEntity.java deleted file mode 100644 index 8cfa4dd486..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatdataset/entity/DcatDatasetEntity.java +++ /dev/null @@ -1,34 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatdataset.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.dcat.dcatdataset.entities.DcatDataset; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(collection = "dcat_dataset") -public class DcatDatasetEntity { - @Id - private final String collectionName; - private final String model; - - public DcatDatasetEntity(String collectionName, String model) { - this.collectionName = collectionName; - this.model = model; - } - - public String getCollectionName() { - return collectionName; - } - - public String getModel() { - return model; - } - - public DcatDataset toDcatDataset() { - return new DcatDataset(collectionName, RDFParser.fromString(model) - .lang(Lang.TURTLE) - .toModel()); - } - -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/DcatCatalogMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/DcatCatalogMigration.java deleted file mode 100644 index 7279d7e24f..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/DcatCatalogMigration.java +++ /dev/null @@ -1,56 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatserver; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatserver.entity.DcatCatalogEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatserver.service.DcatCatalogEntityConverter; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class DcatCatalogMigration { - @Bean - public MongoItemReader dcatCatalogEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatserver.entity.DcatCatalogEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor dcatCatalogEntityProcessor(DcatCatalogEntityConverter mapper) { - return noSQLData -> mapper.fromDcatServer(noSQLData.toDcatServer()); - } - - @Bean - public JpaItemWriter dcatCatalogEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationDcatCatalog") - public Step migrationDcatCatalog(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationDcatCatalog", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/entity/DcatCatalogEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/entity/DcatCatalogEntity.java deleted file mode 100644 index d8ab3aa4c0..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/dcatserver/entity/DcatCatalogEntity.java +++ /dev/null @@ -1,33 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatserver.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.dcat.dcatserver.entities.DcatServer; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document("dcat_catalog") -public class DcatCatalogEntity { - @Id - private final String id; - private final String dcat; - - public DcatCatalogEntity(String id, String dcat) { - this.id = id; - this.dcat = dcat; - } - - public String getId() { - return id; - } - - public String getDcat() { - return dcat; - } - - public DcatServer toDcatServer() { - return new DcatServer(this.id, RDFParser.fromString(this.dcat) - .lang(Lang.TURTLE) - .toModel()); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/EventStreamMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/EventStreamMigration.java deleted file mode 100644 index 85e8e92883..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/EventStreamMigration.java +++ /dev/null @@ -1,57 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.eventstream; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream.entity.EventStreamEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream.service.EventStreamConverter; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class EventStreamMigration { - - @Bean - public MongoItemReader eventStreamEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.eventstream.entity.EventStreamEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor eventStreamEntityProcessor(EventStreamConverter mapper) { - return noSQLData -> mapper.fromEventStream(noSQLData.toEventStream()); - } - - @Bean - public JpaItemWriter eventStreamEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationEventStream") - public Step migrationEventStream(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationEventStream", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/entity/EventStreamEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/entity/EventStreamEntity.java deleted file mode 100644 index d496f8f7d7..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/eventstream/entity/EventStreamEntity.java +++ /dev/null @@ -1,41 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.eventstream.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.EventStream; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(collection = "eventstream") -public class EventStreamEntity { - @Id - private final String id; - private final String timestampPath; - private final String versionOfPath; - private final boolean versionCreationEnabled; - - public EventStreamEntity(String id, String timestampPath, String versionOfPath, boolean versionCreationEnabled) { - this.id = id; - this.timestampPath = timestampPath; - this.versionOfPath = versionOfPath; - this.versionCreationEnabled = versionCreationEnabled; - } - - public String getId() { - return id; - } - - public String getTimestampPath() { - return timestampPath; - } - - public String getVersionOfPath() { - return versionOfPath; - } - - public boolean isVersionCreationEnabled() { - return versionCreationEnabled; - } - - public EventStream toEventStream() { - return new EventStream(id, timestampPath, versionOfPath, versionCreationEnabled); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/ShaclShapeMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/ShaclShapeMigration.java deleted file mode 100644 index d53c2bda9e..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/ShaclShapeMigration.java +++ /dev/null @@ -1,57 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.shaclshape; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.shaclshape.entity.ShaclShapeEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.shaclshape.service.ShaclShapeEntityConverter; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class ShaclShapeMigration { - - @Bean - public MongoItemReader shaclShapeEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.shaclshape.entity.ShaclShapeEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor shaclShapeEntityProcessor(ShaclShapeEntityConverter mapper) { - return noSQLData -> mapper.fromShaclShape(noSQLData.toShaclShape()); - } - - @Bean - public JpaItemWriter shaclShapeEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationShaclShape") - public Step migrationShaclShape(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationShaclShape", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/entity/ShaclShapeEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/entity/ShaclShapeEntity.java deleted file mode 100644 index db172ae47f..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/shaclshape/entity/ShaclShapeEntity.java +++ /dev/null @@ -1,33 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.shaclshape.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.shacl.entities.ShaclShape; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(collection = "shacl_shape") -public class ShaclShapeEntity { - - @Id - private final String id; - - private final String model; - - public ShaclShapeEntity(String id, String model) { - this.id = id; - this.model = model; - } - - public String getId() { - return id; - } - - public String getModel() { - return model; - } - - public ShaclShape toShaclShape() { - return new ShaclShape(id, RDFParser.fromString(model).lang(Lang.TURTLE).toModel()); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/DataServiceMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/DataServiceMigration.java deleted file mode 100644 index 1a9f33b875..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/DataServiceMigration.java +++ /dev/null @@ -1,57 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatdataservice.entity.DataServiceEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatdataservice.service.DcatServiceEntityConverter; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class DataServiceMigration { - - @Bean - public MongoItemReader dataServiceEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view.entity.DataServiceEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor dataServiceEntityProcessor(DcatServiceEntityConverter mapper) { - return noSQLData -> mapper.fromDcatView(noSQLData.toDcatView()); - } - - @Bean - public JpaItemWriter dataServiceEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationDataService") - public Step migrationDataService(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationDataService", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/ViewMigration.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/ViewMigration.java deleted file mode 100644 index c3dfc9c389..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/ViewMigration.java +++ /dev/null @@ -1,57 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view.entity.ViewEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view.service.ViewEntityConverter; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class ViewMigration { - - @Bean - public MongoItemReader viewEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view.entity.ViewEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor viewEntityProcessor(ViewEntityConverter mapper) { - return noSQLData -> mapper.fromView(noSQLData.toView()); - } - - @Bean - public JpaItemWriter viewEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationView") - public Step migrationView(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationView", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/DataServiceEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/DataServiceEntity.java deleted file mode 100644 index d4bafce322..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/DataServiceEntity.java +++ /dev/null @@ -1,37 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.DcatView; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(collection = "dcat_dataservice") -public class DataServiceEntity { - - @Id - private final String viewName; - - private final String model; - - public DataServiceEntity(String viewName, String model) { - this.viewName = viewName; - this.model = model; - } - - public String getViewName() { - return viewName; - } - - public String getModel() { - return model; - } - - public DcatView toDcatView() { - return DcatView.from(ViewName.fromString(viewName), RDFParser.fromString(model) - .lang(Lang.NQUADS) - .toModel()); - } - -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/ViewEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/ViewEntity.java deleted file mode 100644 index f86958dc20..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/mongo/view/entity/ViewEntity.java +++ /dev/null @@ -1,53 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentationConfig; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.util.List; - -@Document(collection = "view") -public class ViewEntity { - - @Id - private final String viewName; - private final List retentionPolicies; - private final List fragmentations; - private final int pageSize; - - public ViewEntity(String viewName, List retentionPolicies, - List fragmentations, int pageSize) { - this.viewName = viewName; - this.retentionPolicies = retentionPolicies; - this.fragmentations = fragmentations; - this.pageSize = pageSize; - } - - public String getViewName() { - return viewName; - } - - public List getRetentionPolicies() { - return retentionPolicies; - } - - public List getFragmentations() { - return fragmentations; - } - - public int getPageSize() { - return pageSize; - } - - public ViewSpecification toView() { - List retentionModels = retentionPolicies.stream() - .map(serializedRetentionModel -> RDFParser.fromString(serializedRetentionModel).lang(Lang.NQUADS).toModel()) - .toList(); - return new ViewSpecification(ViewName.fromString(viewName), retentionModels, fragmentations, pageSize); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/FetchMigrationConfig.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/FetchMigrationConfig.java deleted file mode 100644 index 35cdd5e57d..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/FetchMigrationConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fetch; - - -import be.vlaanderen.informatievlaanderen.ldes.server.fetch.postgres.entity.MemberAllocationEntity; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class FetchMigrationConfig { - @Bean - public MongoItemReader memberAllocationReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setCollection(be.vlaanderen.informatievlaanderen.ldes.server.fetch.entity.MemberAllocationEntity.FETCH_ALLOCATION); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.fetch.entity.MemberAllocationEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor memberAllocationEntityProcessor() { - return noSQLData -> new MemberAllocationEntity(noSQLData.getId(), - noSQLData.getCollectionName(), noSQLData.getViewName(), noSQLData.getFragmentId(), noSQLData.getMemberId()); - } - - @Bean - public JpaItemWriter memberAllocationEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationMongoFetch") - public Job migrationMongoFetchJob(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - Step step = new StepBuilder("migrationMemberAllocation", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - - return new JobBuilder("migrationMongoFetch", jobRepository) - .incrementer(new RunIdIncrementer()) - .flow(step) - .end() - .build(); - } -} \ No newline at end of file diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/entity/MemberAllocationEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/entity/MemberAllocationEntity.java deleted file mode 100644 index 59573f419f..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetch/entity/MemberAllocationEntity.java +++ /dev/null @@ -1,50 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fetch.entity; - -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document(MemberAllocationEntity.FETCH_ALLOCATION) -@CompoundIndex(name = "collection_view", def = "{'collectionName' : 1, 'viewName': 1}") -@CompoundIndex(name = "memberId_view", def = "{'collectionName' : 1, 'viewName': 1, 'memberId' : 1}") -public class MemberAllocationEntity { - public static final String FETCH_ALLOCATION = "fetch_allocation"; - @Id - private final String id; - @Indexed - private final String collectionName; - private final String viewName; - @Indexed - private final String fragmentId; - private final String memberId; - - public MemberAllocationEntity(String id, String collectionName, String viewName, String fragmentId, - String memberId) { - this.id = id; - this.collectionName = collectionName; - this.viewName = viewName; - this.fragmentId = fragmentId; - this.memberId = memberId; - } - - public String getId() { - return id; - } - - public String getCollectionName() { - return collectionName; - } - - public String getViewName() { - return viewName; - } - - public String getFragmentId() { - return fragmentId; - } - - public String getMemberId() { - return memberId; - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationMigrationConfig.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationMigrationConfig.java deleted file mode 100644 index 58d6237ce5..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationMigrationConfig.java +++ /dev/null @@ -1,117 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.FragmentEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.SequenceEntity; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class FragmentationMigrationConfig { - - // Fragment Entity - @Bean - public MongoItemReader fragmentEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entity.FragmentEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor fragmentEntityProcessor() { - return noSQLData -> FragmentEntity.fromLdesFragment(noSQLData.toLdesFragment()); - } - - @Bean - public JpaItemWriter fragmentEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationFragmentEntity") - public Step migrationFragmentEntityStep(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationFragmentEntity", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } - - // Sequence Entity - - @Bean - public MongoItemReader sequenceEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entity.SequenceEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor sequenceEntityProcessor() { - return noSQLData -> new SequenceEntity(noSQLData.getViewName(), noSQLData.getLastProcessedSequence()); - } - - @Bean - public JpaItemWriter sequenceEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationSequenceEntity") - public Step migrationSequenceEntityStep(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - return new StepBuilder("migrationFragmentEntity", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - } - - // Job - - @Bean("migrationMongoFragmentation") - public Job memberEntityMigrationJob(JobRepository jobRepository, - @Qualifier("migrationFragmentEntity") Step fragmentEntityStep, - @Qualifier("migrationSequenceEntity") Step sequenceEntityStep - ) { - - return new JobBuilder("migrationMongoFragmentation", jobRepository) - .incrementer(new RunIdIncrementer()) - .flow(fragmentEntityStep) - .next(sequenceEntityStep) - .end() - .build(); - } - - -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/FragmentEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/FragmentEntity.java deleted file mode 100644 index f3f0b37364..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/FragmentEntity.java +++ /dev/null @@ -1,98 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entity; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.time.LocalDateTime; -import java.util.List; - -@SuppressWarnings({"java:S1068", "java:S107"}) -@Document("fragmentation_fragment") -@CompoundIndex(name = "root_of_view", def = "{'root' : 1, 'viewName': 1}") -@CompoundIndex(name = "immutable_with_parent", def = "{'immutable' : 1, 'parentId': 1}") -public class FragmentEntity { - @Id - private String id; - private Boolean root; - @Indexed - private String viewName; - private List fragmentPairs; - private Boolean immutable; - private String parentId; - private Integer nrOfMembersAdded; - private List relations; - @Indexed - private String collectionName; - @Indexed - private LocalDateTime deleteTime; - private LocalDateTime nextUpdateTs; - - public FragmentEntity(String id, Boolean root, String viewName, List fragmentPairs, - Boolean immutable, String parentId, Integer nrOfMembersAdded, - List relations, String collectionName, LocalDateTime deleteTime, LocalDateTime nextUpdateTs) { - this.id = id; - this.root = root; - this.viewName = viewName; - this.fragmentPairs = fragmentPairs; - this.immutable = immutable; - this.parentId = parentId; - this.nrOfMembersAdded = nrOfMembersAdded; - this.relations = relations; - this.collectionName = collectionName; - this.deleteTime = deleteTime; - this.nextUpdateTs = nextUpdateTs; - } - - public FragmentEntity() { - } - - public String getId() { - return id; - } - - public Boolean isImmutable() { - return immutable; - } - - public Fragment toLdesFragment() { - int effectiveNrOfMembersAdded = nrOfMembersAdded == null ? 0 : nrOfMembersAdded; - final var ldesFragmentIdentifier = new LdesFragmentIdentifier(ViewName.fromString(viewName), fragmentPairs); - final var fragment = - new Fragment(ldesFragmentIdentifier, immutable, effectiveNrOfMembersAdded, relations, deleteTime); - fragment.setNextUpdateTs(nextUpdateTs); - return fragment; - } - - public String getViewName() { - return viewName; - } - - public static FragmentEntity fromLdesFragment(Fragment fragment) { - return new FragmentEntity(fragment.getFragmentIdString(), - fragment.isRoot(), - fragment.getFragmentId().getViewName().asString(), - fragment.getFragmentPairs(), - fragment.isImmutable(), - fragment.getParentIdAsString(), - fragment.getNrOfMembersAdded(), - fragment.getRelations(), - fragment.getFragmentId().getViewName().getCollectionName(), - fragment.getDeleteTime(), - fragment.getNextUpdateTs()); - } - - public List getRelations() { - return relations; - } - - public void removeRelation(TreeRelation treeRelation) { - relations.remove(treeRelation); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/SequenceEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/SequenceEntity.java deleted file mode 100644 index 3323edc2ae..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entity/SequenceEntity.java +++ /dev/null @@ -1,26 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entity; - -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -@Document("fragmentation_sequence") -public class SequenceEntity { - - @Id - private final String viewName; - - private final long lastProcessedSequence; - - public SequenceEntity(String viewName, long lastProcessedSequence) { - this.viewName = viewName; - this.lastProcessedSequence = lastProcessedSequence; - } - - public String getViewName() { - return viewName; - } - - public long getLastProcessedSequence() { - return lastProcessedSequence; - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/IngestMigrationConfig.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/IngestMigrationConfig.java deleted file mode 100644 index e95c766adb..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/IngestMigrationConfig.java +++ /dev/null @@ -1,66 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.ingest; - -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.entity.MemberEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.mapper.MemberEntityMapper; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class IngestMigrationConfig { - - @Bean - public MongoItemReader memberEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.ingest.entity.MemberEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor memberEntityProcessor(MemberEntityMapper mapper) { - return noSQLData -> mapper.toMemberEntity(noSQLData.toMember()); - } - - @Bean - public JpaItemWriter memberEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationMongoIngest") - public Job memberEntityMigrationJob(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - Step step = new StepBuilder("migrationMemberEntity", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - - return new JobBuilder("migrationMongoIngest", jobRepository) - .incrementer(new RunIdIncrementer()) - .flow(step) - .end() - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/entity/MemberEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/entity/MemberEntity.java deleted file mode 100644 index 703e7af884..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/entity/MemberEntity.java +++ /dev/null @@ -1,86 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.ingest.entity; - - -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.entities.Member; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParserBuilder; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.time.LocalDateTime; - -@Document("ingest_ldesmember") -@CompoundIndex(name = "collection_seqNr", def = "{'collectionName' : 1, 'sequenceNr': 1}") -@SuppressWarnings("java:S1117") -public class MemberEntity { - - @Id - private final String id; - - @Indexed - private final String collectionName; - private final String versionOf; - private final LocalDateTime timestamp; - private final boolean isInEventSource; - private Long sequenceNr; - private final String transactionId; - private final String model; - - public MemberEntity(String id, String collectionName, String versionOf, LocalDateTime timestamp, boolean isInEventSource, Long sequenceNr, String transactionId, String model) { - this.id = id; - this.collectionName = collectionName; - this.versionOf = versionOf; - this.timestamp = timestamp; - this.isInEventSource = isInEventSource; - this.sequenceNr = sequenceNr; - this.transactionId = transactionId; - this.model = model; - } - - public String getId() { - return id; - } - - public String getCollectionName() { - return collectionName; - } - - public String getVersionOf() { - return versionOf; - } - - public LocalDateTime getTimestamp() { - return timestamp; - } - - public Long getSequenceNr() { - return sequenceNr; - } - - public void setSequenceNr(Long sequenceNr) { - this.sequenceNr = sequenceNr; - } - - public String getTransactionId() { - return transactionId; - } - - public String getModel() { - return model; - } - - public boolean isInEventSource() { - return isInEventSource; - } - - public Member toMember() { - final Model model = RDFParserBuilder.create().fromString(getModel()).lang(Lang.NQUADS).toModel(); - return new Member(getId(), getCollectionName(), getVersionOf(), getTimestamp(), - getSequenceNr(), isInEventSource(), getTransactionId(), model - ); - } - -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/RetentionMigrationConfig.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/RetentionMigrationConfig.java deleted file mode 100644 index 49b8d11b2d..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/RetentionMigrationConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention; - -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.mapper.MemberEntityMapper; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.postgres.entity.MemberPropertiesEntity; -import jakarta.persistence.EntityManagerFactory; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.launch.support.RunIdIncrementer; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.MongoItemReader; -import org.springframework.batch.item.database.JpaItemWriter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -@ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") -public class RetentionMigrationConfig { - - @Bean - public MongoItemReader memberPropertiesEntityReader(MongoTemplate template) { - MongoItemReader reader = new MongoItemReader<>(); - reader.setTemplate(template); - reader.setTargetType(be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.MemberPropertiesEntity.class); - reader.setQuery(new Query()); - return reader; - } - - @Bean - public ItemProcessor memberPropertiesEntityProcessor(MemberEntityMapper mapper) { - return noSQLData -> new MemberPropertiesEntity(noSQLData.getId(), noSQLData.getCollectionName(), - noSQLData.getViews(), noSQLData.isInEventSource(), noSQLData.getVersionOf(), noSQLData.getTimestamp()); - } - - @Bean - public JpaItemWriter memberPropertiesEntityWriter(EntityManagerFactory entityManagerFactory) { - JpaItemWriter writer = new JpaItemWriter<>(); - writer.setEntityManagerFactory(entityManagerFactory); - return writer; - } - - @Bean("migrationMongoRetention") - public Job retentionMigrationJob(JobRepository jobRepository, - PlatformTransactionManager transactionManager, - MongoItemReader reader, - ItemProcessor processor, - JpaItemWriter writer) { - Step step = new StepBuilder("migrationMemberPropertiesEntity", jobRepository) - .chunk(1000, transactionManager) - .reader(reader) - .processor(processor) - .writer(writer) - .build(); - - return new JobBuilder("migrationMongoRetention", jobRepository) - .incrementer(new RunIdIncrementer()) - .flow(step) - .end() - .build(); - } -} diff --git a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberPropertiesEntity.java b/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberPropertiesEntity.java deleted file mode 100644 index fa396fa317..0000000000 --- a/ldes-server-infra-migration/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberPropertiesEntity.java +++ /dev/null @@ -1,60 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.entities; - -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.time.LocalDateTime; -import java.util.Set; - -@Document(MemberPropertiesEntity.NAME) -@CompoundIndex(name = "versionOf_view", def = "{'views' : 1, 'versionOf': 1}") -public class MemberPropertiesEntity { - - public static final String NAME = "retention_member_properties"; - - @Id - private final String id; - @Indexed - private final String collectionName; - @Indexed - private final Set views; - private final boolean isInEventSource; - private final String versionOf; - private final LocalDateTime timestamp; - - public MemberPropertiesEntity(String id, String collectionName, Set views, boolean isInEventSource, String versionOf, - LocalDateTime timestamp) { - this.id = id; - this.collectionName = collectionName; - this.views = views; - this.isInEventSource = isInEventSource; - this.versionOf = versionOf; - this.timestamp = timestamp; - } - - public String getId() { - return id; - } - - public String getCollectionName() { - return collectionName; - } - - public Set getViews() { - return views; - } - - public boolean isInEventSource() { - return isInEventSource; - } - - public String getVersionOf() { - return versionOf; - } - - public LocalDateTime getTimestamp() { - return timestamp; - } -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/AdminMigrationTest.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/AdminMigrationTest.java deleted file mode 100644 index 857d6dd471..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/AdminMigrationTest.java +++ /dev/null @@ -1,215 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatdataset.entity.DcatDatasetEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.dcatserver.entity.DcatCatalogEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.eventstream.entity.EventStreamEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.shaclshape.entity.ShaclShapeEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view.entity.DataServiceEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.mongo.view.entity.ViewEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatdataservice.repository.DataServiceEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatdataset.repository.DcatDatasetEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.dcatserver.repository.DcatCatalogEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream.repository.EventStreamEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.shaclshape.repository.ShaclShapeEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view.repository.ViewEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentationConfig; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.junit.jupiter.api.Test; -import org.springframework.batch.core.Job; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.ActiveProfiles; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@EnableAutoConfiguration() -@SpringBootTest(classes = {MongoTestConfiguration.class, PostgresTestConfiguration.class}) -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes") -@EntityScan("be.vlaanderen.informatievlaanderen.ldes") -@SpringBatchTest -@ActiveProfiles("test") -@EnableJpaRepositories(basePackages = "be.vlaanderen.informatievlaanderen.ldes") -class AdminMigrationTest { - - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - @Qualifier("migrationMongoAdmin") - private Job job; - - @Autowired - private DcatDatasetEntityRepository dcatDatasetEntityRepository; - @Autowired - private DcatCatalogEntityRepository dcatCatalogEntityRepository; - @Autowired - private EventStreamEntityRepository eventStreamEntityRepository; - @Autowired - private ShaclShapeEntityRepository shaclShapeEntityRepository; - @Autowired - private DataServiceEntityRepository dataServiceEntityRepository; - @Autowired - private ViewEntityRepository viewEntityRepository; - - // Examples - private final String exampleModelTtlString; - private final String exampleModelNqString; - private final Model exampleModel; - private final String retentionPolicy; - private final String timestampPath = "https://w3id.org/ldes#timestampPath"; - private final String versionOfPath = "https://w3id.org/ldes#versionOfPath"; - - public AdminMigrationTest() throws IOException { - retentionPolicy = Files.readString(Path.of("src/test/resources/admin/view/retentionpolicy.nq")); - exampleModelTtlString = Files.readString(Path.of("src/test/resources/randomModel.ttl")); - exampleModelNqString = Files.readString(Path.of("src/test/resources/randomModel.nq")); - exampleModel = RDFParser.fromString(exampleModelTtlString).lang(Lang.TTL).toModel(); - } - - - @Test - void testBatchJob() throws Exception { - saveDcatDatasets(); - saveDcatCatalogs(); - saveEventStreams(); - saveShaclShapes(); - saveDataServices(); - saveViews(); - - jobLauncherTestUtils.setJob(job); - jobLauncherTestUtils.launchJob(); - - verifyDcatDatasets(); - verifyDcatCatalogs(); - verifyEventStreams(); - verifyShaclShapes(); - verifyDataServices(); - verifyViews(); - } - - private void saveDcatDatasets() { - mongoTemplate.save(new DcatDatasetEntity("collection1", exampleModelTtlString)); - mongoTemplate.save(new DcatDatasetEntity("collection2", exampleModelTtlString)); - mongoTemplate.save(new DcatDatasetEntity("collection3", exampleModelTtlString)); - mongoTemplate.save(new DcatDatasetEntity("collection3", exampleModelTtlString)); - } - - private void saveDcatCatalogs() { - mongoTemplate.save(new DcatCatalogEntity("catalog1", exampleModelTtlString)); - mongoTemplate.save(new DcatCatalogEntity("catalog2", exampleModelTtlString)); - mongoTemplate.save(new DcatCatalogEntity("catalog3", exampleModelTtlString)); - mongoTemplate.save(new DcatCatalogEntity("catalog3", exampleModelTtlString)); - } - - private void saveEventStreams() { - mongoTemplate.save(new EventStreamEntity("es1", timestampPath, versionOfPath, true)); - mongoTemplate.save(new EventStreamEntity("es2", timestampPath, versionOfPath, true)); - mongoTemplate.save(new EventStreamEntity("es3", timestampPath, versionOfPath, true)); - mongoTemplate.save(new EventStreamEntity("es3", timestampPath, versionOfPath, true)); - } - - private void saveShaclShapes() { - mongoTemplate.save(new ShaclShapeEntity("shacl1", exampleModelTtlString)); - mongoTemplate.save(new ShaclShapeEntity("shacl2", exampleModelTtlString)); - mongoTemplate.save(new ShaclShapeEntity("shacl3", exampleModelTtlString)); - mongoTemplate.save(new ShaclShapeEntity("shacl3", exampleModelTtlString)); - } - - private void saveDataServices() { - mongoTemplate.save(new DataServiceEntity("es1/view1", exampleModelNqString)); - mongoTemplate.save(new DataServiceEntity("es1/view2", exampleModelNqString)); - mongoTemplate.save(new DataServiceEntity("es1/view3", exampleModelNqString)); - mongoTemplate.save(new DataServiceEntity("es1/view3", exampleModelNqString)); - } - - private void saveViews() { - var fragmentationConfig = new FragmentationConfig(); - fragmentationConfig.setName("fragmentationConfig"); - fragmentationConfig.setConfig(Map.of("pageNumber", "1")); - - mongoTemplate.save(new ViewEntity("es1/view1", List.of(retentionPolicy), List.of(fragmentationConfig), 10)); - mongoTemplate.save(new ViewEntity("es1/view2", List.of(retentionPolicy), List.of(fragmentationConfig), 10)); - mongoTemplate.save(new ViewEntity("es1/view3", List.of(retentionPolicy), List.of(fragmentationConfig), 10)); - mongoTemplate.save(new ViewEntity("es1/view3", List.of(retentionPolicy), List.of(fragmentationConfig), 10)); - } - - private void verifyDcatDatasets() { - var entries = dcatDatasetEntityRepository.findAll(); - assertEquals(3, entries.size()); - - assertTrue(exampleModel.isIsomorphicWith(RDFParser.fromString(entries.get(0).getModel()) - .lang(Lang.TURTLE).toModel())); - } - - private void verifyDcatCatalogs() { - var entries = dcatCatalogEntityRepository.findAll(); - assertEquals(3, entries.size()); - - assertTrue(exampleModel.isIsomorphicWith(RDFParser.fromString(entries.get(0).getDcat()) - .lang(Lang.TURTLE).toModel())); - } - - private void verifyEventStreams() { - var entries = eventStreamEntityRepository.findAll(); - assertEquals(3, entries.size()); - - var es1 = entries.get(0); - assertEquals("es1", es1.getId()); - assertEquals(timestampPath, es1.getTimestampPath()); - assertEquals(versionOfPath, es1.getVersionOfPath()); - assertTrue(es1.isVersionCreationEnabled()); - } - - private void verifyShaclShapes() { - var entries = shaclShapeEntityRepository.findAll(); - assertEquals(3, entries.size()); - - assertTrue(exampleModel.isIsomorphicWith(RDFParser.fromString(entries.get(0).getModel()) - .lang(Lang.TURTLE).toModel())); - } - - private void verifyDataServices() { - var entries = dataServiceEntityRepository.findAll(); - assertEquals(3, entries.size()); - - assertTrue(exampleModel.isIsomorphicWith(RDFParser.fromString(entries.get(0).getModel()) - .lang(Lang.TURTLE).toModel())); - } - - private void verifyViews() { - var entries = viewEntityRepository.findAll(); - assertEquals(3, entries.size()); - - var view = entries.get(0); - assertEquals("es1/view1", view.getViewName()); - var retentionPolicyModel = RDFParser.fromString(retentionPolicy) - .lang(Lang.NQUADS) - .toModel(); - var persistedRetentionPolicyModel = RDFParser.fromString(view.getRetentionPolicies().get(0)) - .lang(Lang.TTL) - .toModel(); - assertTrue(retentionPolicyModel.isIsomorphicWith(persistedRetentionPolicyModel)); - assertEquals("1", view.getFragmentations().get(0).getConfig().get("pageNumber")); - assertEquals(10, view.getPageSize()); - } -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FetchMigrationTest.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FetchMigrationTest.java deleted file mode 100644 index dfb217d8ec..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FetchMigrationTest.java +++ /dev/null @@ -1,68 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import be.vlaanderen.informatievlaanderen.ldes.server.fetch.entity.MemberAllocationEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.fetch.postgres.repository.AllocationEntityRepository; -import org.junit.jupiter.api.Test; -import org.springframework.batch.core.Job; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.ActiveProfiles; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@EnableAutoConfiguration() -@SpringBootTest(classes = {MongoTestConfiguration.class, PostgresTestConfiguration.class}) -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes") -@EntityScan("be.vlaanderen.informatievlaanderen.ldes") -@SpringBatchTest -@ActiveProfiles("test") -@EnableJpaRepositories(basePackages = "be.vlaanderen.informatievlaanderen.ldes") -class FetchMigrationTest { - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - @Qualifier("migrationMongoFetch") - private Job job; - - @Autowired - private AllocationEntityRepository repository; - - @Test - void testBatchJob() throws Exception { - String es = "es"; - String view = "v"; - String fragment = "/%s/%s?page=1".formatted(es, view); - String memberId = "%s/https://ex.com/1".formatted(es); - mongoTemplate.save(new MemberAllocationEntity("%s/%s/%s".formatted(es, memberId, fragment), es, view, fragment, memberId)); - memberId = "%s/https://ex.com/2".formatted(es); - mongoTemplate.save(new MemberAllocationEntity("%s/%s/%s".formatted(es, memberId, fragment), es, view, fragment, memberId)); - memberId = "%s/https://ex.com/3".formatted(es); - mongoTemplate.save(new MemberAllocationEntity("%s/%s/%s".formatted(es, memberId, fragment), es, view, fragment, memberId)); - mongoTemplate.save(new MemberAllocationEntity("%s/%s/%s".formatted(es, memberId, fragment), es, view, fragment, memberId)); - - jobLauncherTestUtils.setJob(job); - jobLauncherTestUtils.launchJob(); - - var entries = repository.findAll(); - assertEquals(3, entries.size()); - - var lastEntry = entries.get(2); - - assertEquals(memberId, lastEntry.getMemberId()); - assertEquals(view, lastEntry.getViewName()); - assertEquals(fragment, lastEntry.getFragmentId()); - assertEquals(es, lastEntry.getCollectionName()); - } -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentMigrationTest.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentMigrationTest.java deleted file mode 100644 index 1d4f6686dc..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentMigrationTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entity.FragmentEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entity.SequenceEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.FragmentEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.SequenceEntityRepository; -import org.junit.jupiter.api.Test; -import org.springframework.batch.core.Job; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.ActiveProfiles; - -import java.time.LocalDateTime; -import java.util.List; - -import static be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier.fromFragmentId; -import static org.junit.jupiter.api.Assertions.assertEquals; - -@EnableAutoConfiguration() -@SpringBootTest(classes = {MongoTestConfiguration.class, PostgresTestConfiguration.class}) -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes") -@EntityScan("be.vlaanderen.informatievlaanderen.ldes") -@SpringBatchTest -@ActiveProfiles("test") -@EnableJpaRepositories(basePackages = "be.vlaanderen.informatievlaanderen.ldes") -class FragmentMigrationTest { - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - @Qualifier("migrationMongoFragmentation") - private Job job; - - @Autowired - private FragmentEntityRepository fragmentEntityRepository; - @Autowired - private SequenceEntityRepository sequenceEntityRepository; - - private static final String ES = "es"; - private static final String VIEW = "v"; - - @Test - void testBatchJob() throws Exception { - saveFragments(); - saveSequences(); - - jobLauncherTestUtils.setJob(job); - jobLauncherTestUtils.launchJob(); - - verifyFragments(); - verifySequences(); - } - - private void saveFragments() { - LdesFragmentIdentifier fragmentId = fromFragmentId("/%s/%s".formatted(ES, VIEW)); - mongoTemplate.save(fragmentEntity(fragmentId)); - fragmentId = fromFragmentId("/%s/%s?foo=bar".formatted(ES, VIEW)); - mongoTemplate.save(fragmentEntity(fragmentId)); - fragmentId = fromFragmentId("/%s/%s?foo=pub".formatted(ES, VIEW)); - mongoTemplate.save(fragmentEntity(fragmentId)); - mongoTemplate.save(fragmentEntity(fragmentId)); - } - - private void saveSequences() { - mongoTemplate.save(new SequenceEntity("/es/v1", 1)); - mongoTemplate.save(new SequenceEntity("/es/v2", 1)); - mongoTemplate.save(new SequenceEntity("/es/v3", 1)); - mongoTemplate.save(new SequenceEntity("/es/v3", 1)); - } - - private void verifyFragments() { - var entries = fragmentEntityRepository.findAll(); - - assertEquals(3, entries.size()); - - var entry = entries.get(0); - assertEquals("/%s/%s".formatted(ES, VIEW), entry.getId()); - assertEquals(fromFragmentId("/%s/%s".formatted(ES, VIEW)).getViewName().asString(), entry.getViewName()); - - var relation = entry.getRelations().get(0); - assertEquals(exampleRelation().relation(), relation.getRelation()); - assertEquals(exampleRelation().treeNode().toString(), relation.getTreeNode()); - assertEquals(exampleRelation().treePath(), relation.getTreePath()); - assertEquals(exampleRelation().treeValue(), relation.getTreeValue()); - assertEquals(exampleRelation().treeValueType(), relation.getTreeValueType()); - } - - private void verifySequences() { - var entries = sequenceEntityRepository.findAll(); - assertEquals(3, entries.size()); - - assertEquals(1, entries.get(0).getLastProcessedSequence()); - assertEquals("/es/v1", entries.get(0).getViewName()); - } - - private FragmentEntity fragmentEntity(LdesFragmentIdentifier fragmentId) { - return new FragmentEntity(fragmentId.toString(), true, - fragmentId.getViewName().asString(), fragmentId.getFragmentPairs(), - false, fragmentId.getParentId().toString(), 0, - List.of(exampleRelation()), fragmentId.getViewName().getCollectionName(), - LocalDateTime.now(), LocalDateTime.now()); - } - - private TreeRelation exampleRelation() { - return new TreeRelation("path", fromFragmentId("/%s/%s".formatted(ES, VIEW)), "value", "valueType", "relation"); - } -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/IngestMigrationTest.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/IngestMigrationTest.java deleted file mode 100644 index ba949b787c..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/IngestMigrationTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.entity.MemberEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; -import org.junit.jupiter.api.Test; -import org.springframework.batch.core.Job; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.ActiveProfiles; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@EnableAutoConfiguration() -@SpringBootTest(classes = {MongoTestConfiguration.class, PostgresTestConfiguration.class}) -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes") -@EntityScan("be.vlaanderen.informatievlaanderen.ldes") -@SpringBatchTest -@ActiveProfiles("test") -@EnableJpaRepositories(basePackages = "be.vlaanderen.informatievlaanderen.ldes") -class IngestMigrationTest { - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - @Qualifier("migrationMongoIngest") - private Job job; - - @Autowired - private MemberEntityRepository repository; - - private final String exampleModelNqString; - private final Model exampleModel; - - public IngestMigrationTest() throws IOException { - this.exampleModelNqString = Files.readString(Path.of("src/test/resources/randomModel.nq")); - this.exampleModel = RDFParser.fromString(exampleModelNqString) - .lang(Lang.NQUADS) - .toModel(); - - } - - @Test - void testBatchJob() throws Exception { - String es = "es"; - String memberId = "%s/https://ex.com/1".formatted(es); - String versionOf = "https://example.com/John-Doe"; - LocalDateTime timestamp = LocalDateTime.parse("2024-05-14T11:08:30.217"); - mongoTemplate.save(new MemberEntity(memberId, es, versionOf, timestamp, true,1L, "1", exampleModelNqString)); - memberId = "%s/https://ex.com/2".formatted(es); - mongoTemplate.save(new MemberEntity(memberId, es, versionOf, timestamp, true,2L, "1", exampleModelNqString)); - memberId = "%s/https://ex.com/3".formatted(es); - mongoTemplate.save(new MemberEntity(memberId, es, versionOf, timestamp, true,3L, "1", exampleModelNqString)); - mongoTemplate.save(new MemberEntity(memberId, es, versionOf, timestamp, true,3L, "1", exampleModelNqString)); - - jobLauncherTestUtils.setJob(job); - jobLauncherTestUtils.launchJob(); - - var entries = repository.findAll(); - assertEquals(3, entries.size()); - - var lastEntry = entries.get(2); - - assertEquals(memberId, lastEntry.getId()); - assertEquals(es, lastEntry.getCollectionName()); - assertEquals("1", lastEntry.getTransactionId()); - assertEquals(timestamp, lastEntry.getTimestamp()); - assertEquals(versionOf, lastEntry.getVersionOf()); - assertEquals(3L, lastEntry.getSequenceNr()); - assertTrue(lastEntry.isInEventSource()); - - ByteArrayInputStream inputStream = new ByteArrayInputStream(lastEntry.getModel()); - Model returnedModel = RDFParser.source(inputStream).lang(Lang.RDFPROTO).toModel(); - - assertTrue(exampleModel.isIsomorphicWith(returnedModel)); - } -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoTestConfiguration.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoTestConfiguration.java deleted file mode 100644 index 52e0851cd9..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/MongoTestConfiguration.java +++ /dev/null @@ -1,36 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.DependsOn; -import org.springframework.data.mongodb.MongoDatabaseFactory; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory; -import org.testcontainers.containers.MongoDBContainer; - -@SpringBootConfiguration -public class MongoTestConfiguration { - - @Bean - public MongoDBContainer mongoDBContainer() { - var container = new MongoDBContainer("mongo:latest") - .withExposedPorts(27017); - container.start(); - return container; - } - - @Bean - @DependsOn("mongoDBContainer") - @ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") - public MongoDatabaseFactory mongoDatabaseFactory(MongoDBContainer mongo) { - return new SimpleMongoClientDatabaseFactory(mongo.getConnectionString() + "/test"); - } - - @Bean - @DependsOn("mongoDatabaseFactory") - @ConditionalOnProperty(name = "ldes-server.migrate-mongo", havingValue = "true") - public MongoTemplate mongoTemplate(MongoDatabaseFactory mongoDbFactory) { - return new MongoTemplate(mongoDbFactory); - } -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/PostgresTestConfiguration.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/PostgresTestConfiguration.java deleted file mode 100644 index f0785506b4..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/PostgresTestConfiguration.java +++ /dev/null @@ -1,46 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.DependsOn; -import org.springframework.jdbc.datasource.DriverManagerDataSource; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.transaction.PlatformTransactionManager; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.containers.wait.strategy.Wait; - -import javax.sql.DataSource; - -@SpringBootConfiguration -public class PostgresTestConfiguration { - @Bean - public PostgreSQLContainer postgresqlContainer() { - var container = new PostgreSQLContainer("postgres:13.1") - .withDatabaseName("test") - .withUsername("postgres") - .withPassword("postgres") - .waitingFor(Wait.forListeningPort()); - - container.start(); - return (PostgreSQLContainer) container; - } - - @Bean - @DependsOn("postgresqlContainer") - public DataSource dataSource(PostgreSQLContainer container) { - DriverManagerDataSource dataSource = new DriverManagerDataSource(); - dataSource.setUrl(container.getJdbcUrl()); - dataSource.setUsername(container.getUsername()); - dataSource.setPassword(container.getPassword()); - return dataSource; - } - - @Bean - @DependsOn("dataSource") - public PlatformTransactionManager transactionManager(DataSource dataSource) { - final JpaTransactionManager tm = new JpaTransactionManager(); - tm.setDataSource(dataSource); - return tm; - } - -} diff --git a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/RetentionMigrationTest.java b/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/RetentionMigrationTest.java deleted file mode 100644 index 22440250a8..0000000000 --- a/ldes-server-infra-migration/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/RetentionMigrationTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server; - -import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.MemberPropertiesEntity; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.postgres.repository.MemberPropertiesEntityRepository; -import org.junit.jupiter.api.Test; -import org.springframework.batch.core.Job; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.ActiveProfiles; - -import java.time.LocalDateTime; -import java.util.List; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@EnableAutoConfiguration() -@SpringBootTest(classes = {MongoTestConfiguration.class, PostgresTestConfiguration.class}) -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes") -@EntityScan("be.vlaanderen.informatievlaanderen.ldes") -@SpringBatchTest -@ActiveProfiles("test") -class RetentionMigrationTest { - @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; - - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - @Qualifier("migrationMongoRetention") - private Job job; - - @Autowired - private MemberPropertiesEntityRepository repository; - - @Test - void testBatchJob() throws Exception { - String es = "es"; - String memberId = "%s/https://ex.com/1".formatted(es); - String versionOf = "https://example.com/John-Doe"; - LocalDateTime timestamp = LocalDateTime.parse("2024-05-14T11:08:30.217"); - mongoTemplate.save(new MemberPropertiesEntity(memberId, es, Set.of("v1", "v2"), true, versionOf, timestamp)); - memberId = "%s/https://ex.com/2".formatted(es); - mongoTemplate.save(new MemberPropertiesEntity(memberId, es, Set.of("v1", "v2"), true, versionOf, timestamp)); - memberId = "%s/https://ex.com/3".formatted(es); - mongoTemplate.save(new MemberPropertiesEntity(memberId, es, Set.of("v1", "v2"), true, versionOf, timestamp)); - mongoTemplate.save(new MemberPropertiesEntity(memberId, es, Set.of("v1", "v2"), true, versionOf, timestamp)); - - jobLauncherTestUtils.setJob(job); - jobLauncherTestUtils.launchJob(); - - var entries = repository.findAll(); - assertEquals(3, entries.size()); - - var lastEntry = entries.get(2); - - assertEquals(memberId, lastEntry.getId()); - assertEquals(es, lastEntry.getCollectionName()); - assertEquals(timestamp, lastEntry.getTimestamp()); - assertEquals(versionOf, lastEntry.getVersionOf()); - assertTrue(lastEntry.isInEventSource()); - assertTrue(lastEntry.getViews().containsAll(List.of("v1", "v2"))); - } -} diff --git a/ldes-server-infra-migration/src/test/resources/admin/view/retentionpolicy.nq b/ldes-server-infra-migration/src/test/resources/admin/view/retentionpolicy.nq deleted file mode 100644 index b9e0b758e4..0000000000 --- a/ldes-server-infra-migration/src/test/resources/admin/view/retentionpolicy.nq +++ /dev/null @@ -1,2 +0,0 @@ -_:genid1 . -_:genid1 "PT2M"^^ . \ No newline at end of file diff --git a/ldes-server-infra-migration/src/test/resources/admin/view/view_valid.ttl b/ldes-server-infra-migration/src/test/resources/admin/view/view_valid.ttl deleted file mode 100644 index 21e3698051..0000000000 --- a/ldes-server-infra-migration/src/test/resources/admin/view/view_valid.ttl +++ /dev/null @@ -1,41 +0,0 @@ -@prefix ldes: . -@prefix tree: . -@prefix dc: . -@prefix host: . -@prefix server: . -@prefix viewName: . -@prefix dcat: . -@prefix skos: . -@prefix xsd: . -@prefix rdfs: . - -server:viewName a tree:Node, rdfs:Resource ; - tree:viewDescription viewName:description . - -viewName:description - a dcat:DataService , tree:ViewDescription ; - tree:pageSize "100"^^; - tree:fragmentationStrategy - ([ a tree:ExampleFragmentation ; - tree:pageSize "100" ; - tree:property "example/property" - ]) ; - dc:description "Geospatial fragmentation for my LDES"@en ; - dc:identifier "http://localhost:8080/collection/viewName"^^rdfs:Literal ; - dc:title "My geo-spatial view"@en ; - dc:license - [ a dc:LicenseDocument ; - dc:type - [ a skos:Concept ; - skos:prefLabel "some public license"@en - ] - ] ; - ldes:retentionPolicy [ - a ldes:DurationAgoPolicy ; - tree:value "PT2M"^^xsd:duration ; - ] ; - dcat:endpointDescription ; - dcat:endpointURL server:viewName ; - dcat:servesDataset host:collection . - - a rdfs:Resource . \ No newline at end of file diff --git a/ldes-server-infra-migration/src/test/resources/application-test.yml b/ldes-server-infra-migration/src/test/resources/application-test.yml deleted file mode 100644 index d756be0d3a..0000000000 --- a/ldes-server-infra-migration/src/test/resources/application-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -ldes-server: - migrate-mongo: "true" - host-name: abc - -spring: - main: - allow-bean-definition-overriding: true - data: - mongodb: - uri: mongodb://localhost:27017/test - datasource: - url: jdbc:postgresql://localhost:5432/test - username: admin - password: admin - jpa: - hibernate: - ddl-auto: update - - batch: - jdbc: - initialize-schema: always - job: - enabled: false - diff --git a/ldes-server-infra-migration/src/test/resources/randomModel.nq b/ldes-server-infra-migration/src/test/resources/randomModel.nq deleted file mode 100644 index 6c5dba64de..0000000000 --- a/ldes-server-infra-migration/src/test/resources/randomModel.nq +++ /dev/null @@ -1,23 +0,0 @@ -_:genid1 "1 cup ice cubes" . -_:genid1 "1 tablespoons white sugar" . -_:genid1 "1/2 cup club soda" . -_:genid1 "1/2 lime, juiced with pulp" . -_:genid1 "12 fresh mint leaves" . -_:genid1 "2 fluid ounces white rum" . -_:genid1 _:genid2 . -_:genid1 _:genid3 . -_:genid1 _:genid4 . -_:genid1 _:genid5 . -_:genid1 _:genid6 . -_:genid1 "Mojito" . -_:genid1 "1 cocktail" . -_:genid2 "Crush lime juice, mint and sugar together in glass." . -_:genid2 "1"^^ . -_:genid3 "Fill glass to top with ice cubes." . -_:genid3 "2"^^ . -_:genid4 "Pour white rum over ice." . -_:genid4 "3"^^ . -_:genid5 "Fill the rest of glass with club soda, stir." . -_:genid5 "4"^^ . -_:genid6 "Garnish with a lime wedge." . -_:genid6 "5"^^ . \ No newline at end of file diff --git a/ldes-server-infra-migration/src/test/resources/randomModel.ttl b/ldes-server-infra-migration/src/test/resources/randomModel.ttl deleted file mode 100644 index f3af5d1b51..0000000000 --- a/ldes-server-infra-migration/src/test/resources/randomModel.ttl +++ /dev/null @@ -1,22 +0,0 @@ -@prefix v: . - -[] - v:ingredients "1 cup ice cubes", "1 tablespoons white sugar", "1/2 cup club soda", "1/2 lime, juiced with pulp", "12 fresh mint leaves", "2 fluid ounces white rum" ; - v:instructions [ - v:description "Crush lime juice, mint and sugar together in glass." ; - v:step 1 - ], [ - v:description "Fill glass to top with ice cubes." ; - v:step 2 - ], [ - v:description "Pour white rum over ice." ; - v:step 3 - ], [ - v:description "Fill the rest of glass with club soda, stir." ; - v:step 4 - ], [ - v:description "Garnish with a lime wedge." ; - v:step 5 - ] ; - v:name "Mojito" ; - v:yield "1 cocktail" . \ No newline at end of file diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 5023bd5e50..5e97ff2787 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT pom diff --git a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml index a0b73e2ec3..9f79e4e43a 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT postgres-admin-repository diff --git a/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml b/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml index 05d54daec7..8066214e44 100644 --- a/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT postgres-fetch-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml index 12cc4d2148..838902d53d 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT postgres-fragmentation-repository diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml index 7810eb552e..09c60b4ea4 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT postgres-ingest-repository diff --git a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml index 9eee89d58c..b0c94d2fc0 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT postgres-retention-repository diff --git a/ldes-server-instrumentation/pom.xml b/ldes-server-instrumentation/pom.xml index 3ec55211ec..5db7d8631d 100644 --- a/ldes-server-instrumentation/pom.xml +++ b/ldes-server-instrumentation/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-instrumentation diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index d8cbf54ba8..06da56e16a 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-integration-test diff --git a/ldes-server-port-fetch-rest/pom.xml b/ldes-server-port-fetch-rest/pom.xml index 0bb60ef018..7464ef9bb1 100644 --- a/ldes-server-port-fetch-rest/pom.xml +++ b/ldes-server-port-fetch-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-port-fetch-rest diff --git a/ldes-server-port-fetch/pom.xml b/ldes-server-port-fetch/pom.xml index f8bd54741c..f99c7aa66f 100644 --- a/ldes-server-port-fetch/pom.xml +++ b/ldes-server-port-fetch/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-port-fetch diff --git a/ldes-server-port-ingest-rest/pom.xml b/ldes-server-port-ingest-rest/pom.xml index 52b4da9edb..08b3e3a829 100644 --- a/ldes-server-port-ingest-rest/pom.xml +++ b/ldes-server-port-ingest-rest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-port-ingest-rest diff --git a/ldes-server-port-ingest/pom.xml b/ldes-server-port-ingest/pom.xml index 7f7c78bc15..825d504968 100644 --- a/ldes-server-port-ingest/pom.xml +++ b/ldes-server-port-ingest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-port-ingest diff --git a/ldes-server-retention/pom.xml b/ldes-server-retention/pom.xml index ff21815ccf..e6c5093edb 100644 --- a/ldes-server-retention/pom.xml +++ b/ldes-server-retention/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT ldes-server-retention diff --git a/pom.xml b/pom.xml index 05903c8109..d218e839f3 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT pom @@ -26,7 +26,6 @@ ldes-server-compaction ldes-server-integration-test ldes-server-instrumentation - ldes-server-infra-migration From 6a80179300b7b550a8ff477f8bfadbdab714d0f0 Mon Sep 17 00:00:00 2001 From: pj-cegeka <119848850+pj-cegeka@users.noreply.github.com> Date: Thu, 30 May 2024 13:30:51 +0200 Subject: [PATCH 02/15] Feat:split pagination and fragmentation (#1294) * temp * temp * temp * temp * Fix: fix tests * Fix: fix delete view * feat/ liquibase migration * feat: add tests * feat: add tests * feat: create pagination module * feat: fix liquibase * feat: fix test * feat: fix pom * feat: fix pom * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * feat: fix test * fix: issues --- Dockerfile | 4 +- docker-compose/server.config.yml | 5 +- .../fragmentation/BucketisedMemberSaver.java | 9 ++ .../BucketisedMemberSaverImpl.java | 26 ++++++ .../fragmentation/FragmentationStrategy.java | 7 +- .../FragmentationStrategyCollectionImpl.java | 13 ++- .../FragmentationStrategyDecorator.java | 11 ++- .../FragmentationStrategyExecutor.java | 20 ++-- .../FragmentationStrategyImpl.java | 27 ++---- .../entities/BucketisedMember.java | 6 ++ .../FragmentationStrategyCreatorImpl.java | 17 +--- ...gmentationStrategyExecutorCreatorImpl.java | 8 +- .../BucketisedMemberRepository.java | 13 +++ .../FragmentSequenceRepository.java | 2 +- ...agmentationStrategyCollectionImplTest.java | 14 ++- .../FragmentationStrategyCreatorImplTest.java | 35 ++----- .../FragmentationStrategyDecoratorTest.java | 5 +- .../FragmentationStrategyExecutorTest.java | 32 ++++--- .../FragmentationStrategyImplTest.java | 33 ++++--- .../entities/FragmentSequenceTest.java | 1 + .../GeospatialFragmentationStrategy.java | 19 ++-- .../GeospatialFragmentationStrategyTest.java | 14 ++- .../pagination/PaginationStrategy.java | 46 --------- .../PaginationStrategyAutoConfiguration.java | 20 ---- .../pagination/PaginationStrategyWrapper.java | 48 ---------- .../pagination/config/PaginationConfig.java | 4 - .../pagination/PaginationStrategyTest.java | 86 ----------------- .../PaginationStrategyWrapperTest.java | 31 ------- .../ReferenceFragmentationStrategy.java | 19 ++-- .../ReferenceFragmentationStrategyTest.java | 8 +- ...rchicalTimeBasedFragmentationStrategy.java | 12 ++- ...calTimeBasedFragmentationStrategyTest.java | 8 +- ldes-fragmentisers/pom.xml | 1 - ldes-server-application/pom.xml | 20 +--- .../FragmentationConfigCompaction.java | 4 +- .../compaction/CompactionIntegrationTest.java | 8 -- .../fragmentation/MemberBucketisedEvent.java | 6 ++ .../domain/model}/FragmentSequence.java | 4 +- ldes-server-infra-postgres/pom.xml | 2 + .../BucketisedMemberPostgresRepository.java | 46 +++++++++ .../FragmentSequencePostgresRepository.java | 2 +- .../postgres/entity/MemberBucketEntity.java | 54 +++++++++++ .../mapper/MemberBucketEntityMapper.java | 20 ++++ .../postgres/mapper/SequenceEntityMapper.java | 2 +- .../repository/FragmentEntityRepository.java | 54 +++++------ .../MemberBucketEntityRepository.java | 12 +++ .../repository/SequenceEntityRepository.java | 2 +- .../PostgresFragmentationIntegrationTest.java | 6 +- .../bucketisedmember/BucketRepositoryIT.java | 12 +++ .../BucketRepositorySteps.java | 85 +++++++++++++++++ .../FragmentSequenceRepositorySteps.java | 2 +- .../resources/features/bucket/bucket.feature | 34 +++++++ .../postgres-liquibase/pom.xml | 21 +++++ .../resources/db/changelog/3_0_0/admin.xml | 70 ++++++++++++++ .../resources/db/changelog/3_0_0/fetch.xml | 38 ++++++++ .../db/changelog/3_0_0/fragmentation.xml | 66 +++++++++++++ .../resources/db/changelog/3_0_0/ingest.xml | 34 +++++++ .../resources/db/changelog/3_0_0/master.xml | 20 ++++ .../db/changelog/3_0_0/retention.xml | 41 ++++++++ .../db/changelog/3_0_1/bucketisation.sql | 3 + .../db/changelog/3_0_1/bucketisation.xml | 34 +++++++ .../resources/db/changelog/3_0_1/master.xml | 9 ++ .../changelog/3_0_1/pagination_sequence.sql | 2 + .../changelog/3_0_1/pagination_sequence.xml | 20 ++++ .../main/resources/db/changelog/master.xml | 7 ++ .../postgres-pagination-repository/pom.xml | 32 +++++++ .../PaginationSequencePostgresRepository.java | 51 ++++++++++ .../entity/PaginationSequenceEntity.java | 30 ++++++ .../PaginationSequenceEntityMapper.java | 21 +++++ .../PaginationSequenceEntityRepository.java | 11 +++ .../PaginationSequenceRepositorySteps.java | 49 ++++++++++ .../PostgresPaginationIntegrationTest.java | 72 ++++++++++++++ .../resources/pagination-sequence.feature | 36 +++++++ ldes-server-integration-test/pom.xml | 8 +- .../ldes/server/FragmentationSteps.java | 6 +- .../ldes/server/LdesServerSteps.java | 6 +- .../pom.xml | 32 +++++-- .../pagination/MemberPaginationService.java | 93 +++++++++++++++++++ .../MemberPaginationServiceCreator.java | 56 +++++++++++ .../server/pagination/PaginationService.java | 78 ++++++++++++++++ .../pagination/config/PaginationConfig.java | 4 + .../config/PaginationProperties.java | 2 +- .../constants/PaginationConstants.java | 2 +- .../PaginationSequenceRepository.java | 16 ++++ .../pagination/services/OpenPageProvider.java | 26 ++++-- .../pagination/services/PageCreator.java | 14 ++- .../MemberPaginationServiceCreatorTest.java | 48 ++++++++++ .../MemberPaginationServiceTest.java | 65 +++++++++++++ .../pagination/PaginationServiceTest.java | 80 ++++++++++++++++ .../services/OpenPageProviderTest.java | 36 +++---- .../pagination/services/PageCreatorTest.java | 6 +- pom.xml | 1 + 92 files changed, 1721 insertions(+), 504 deletions(-) create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaver.java create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaverImpl.java create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketisedMember.java create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategy.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyAutoConfiguration.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapper.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationConfig.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyTest.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapperTest.java create mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberBucketisedEvent.java rename {ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities => ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model}/FragmentSequence.java (73%) create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketisedMemberPostgresRepository.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/MemberBucketEntityMapper.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/MemberBucketEntityRepository.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositoryIT.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositorySteps.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/features/bucket/bucket.feature create mode 100644 ldes-server-infra-postgres/postgres-liquibase/pom.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/admin.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fetch.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fragmentation.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/ingest.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/master.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/retention.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/master.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/pom.xml create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSequencePostgresRepository.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PaginationSequenceEntity.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/PaginationSequenceEntityMapper.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PaginationSequenceEntityRepository.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationSequenceRepositorySteps.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PostgresPaginationIntegrationTest.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/pagination-sequence.feature rename {ldes-fragmentisers/ldes-fragmentisers-pagination => ldes-server-pagination}/pom.xml (57%) create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationService.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreator.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationService.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationConfig.java rename {ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers => ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server}/pagination/config/PaginationProperties.java (71%) rename {ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers => ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server}/pagination/constants/PaginationConstants.java (67%) create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java rename {ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers => ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server}/pagination/services/OpenPageProvider.java (52%) rename {ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers => ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server}/pagination/services/PageCreator.java (81%) create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreatorTest.java create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceTest.java create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationServiceTest.java rename {ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers => ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server}/pagination/services/OpenPageProviderTest.java (74%) rename {ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers => ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server}/pagination/services/PageCreatorTest.java (88%) diff --git a/Dockerfile b/Dockerfile index e6e6a7b113..8583336d72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,8 +21,8 @@ COPY --from=app-stage ldes-server-infra-postgres/postgres-ingest-repository/targ COPY --from=app-stage ldes-server-infra-postgres/postgres-fetch-repository/target/postgres-fetch-repository-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-infra-postgres/postgres-retention-repository/target/postgres-retention-repository-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-infra-postgres/postgres-fragmentation-repository/target/postgres-fragmentation-repository-jar-with-dependencies.jar ./lib/ +COPY --from=app-stage ldes-server-infra-postgres/postgres-pagination-repository/target/postgres-pagination-repository-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-infra-postgres/postgres-admin-repository/target/postgres-admin-repository-jar-with-dependencies.jar ./lib/ -COPY --from=app-stage ldes-server-infra-migration/target/ldes-server-infra-migration-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-port-ingest-rest/target/ldes-server-port-ingest-rest-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-port-ingest/target/ldes-server-port-ingest-jar-with-dependencies.jar ./lib/ @@ -32,8 +32,8 @@ COPY --from=app-stage ldes-server-admin/target/ldes-server-admin-jar-with-depend COPY --from=app-stage ldes-fragmentisers/ldes-fragmentisers-common/target/ldes-fragmentisers-common-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-fragmentisers/ldes-fragmentisers-geospatial/target/ldes-fragmentisers-geospatial-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/target/ldes-fragmentisers-timebased-hierarchical-jar-with-dependencies.jar ./lib/ -COPY --from=app-stage ldes-fragmentisers/ldes-fragmentisers-pagination/target/ldes-fragmentisers-pagination-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-fragmentisers/ldes-fragmentisers-reference/target/ldes-fragmentisers-reference-jar-with-dependencies.jar ./lib/ +COPY --from=app-stage ldes-server-pagination/target/ldes-server-pagination-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-retention/target/ldes-server-retention-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-compaction/target/ldes-server-compaction-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldes-server-instrumentation/target/ldes-server-instrumentation-jar-with-dependencies.jar ./lib/ diff --git a/docker-compose/server.config.yml b/docker-compose/server.config.yml index 15b77707c3..77481afd2f 100644 --- a/docker-compose/server.config.yml +++ b/docker-compose/server.config.yml @@ -27,9 +27,10 @@ spring: username: admin password: admin jpa: - hibernate: - ddl-auto: update database: postgresql + liquibase: + change-log: classpath:/db/changelog/master.xml + #server: # servlet: # context-path: /ldes \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaver.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaver.java new file mode 100644 index 0000000000..7688154ca5 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaver.java @@ -0,0 +1,9 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; + +import java.util.List; + +public interface BucketisedMemberSaver { + void save(List members); +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaverImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaverImpl.java new file mode 100644 index 0000000000..87eef8f871 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/BucketisedMemberSaverImpl.java @@ -0,0 +1,26 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.MemberBucketisedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class BucketisedMemberSaverImpl implements BucketisedMemberSaver { + private final BucketisedMemberRepository repository; + private final ApplicationEventPublisher eventPublisher; + + public BucketisedMemberSaverImpl(BucketisedMemberRepository repository, ApplicationEventPublisher eventPublisher) { + this.repository = repository; + this.eventPublisher = eventPublisher; + } + + @Override + public void save(List members) { + repository.insertAll(members); + eventPublisher.publishEvent(new MemberBucketisedEvent(members.getFirst().viewName())); + } +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java index 1334f28a1a..b5aa7578bd 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java @@ -1,9 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import io.micrometer.observation.Observation; -import org.apache.jena.rdf.model.Model; + +import java.util.List; public interface FragmentationStrategy { - void addMemberToFragment(Fragment rootFragmentOfView, String memberId, Model memberModel, Observation parentObservation); + List addMemberToFragment(Fragment rootFragmentOfView, Member member, Observation parentObservation); } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImpl.java index 3beb046b2b..f46c9ba75e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImpl.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImpl.java @@ -7,6 +7,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyExecutorCreatorImpl; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentSequenceRepository; import org.springframework.context.event.EventListener; @@ -25,15 +26,17 @@ public class FragmentationStrategyCollectionImpl implements FragmentationStrateg private final Set fragmentationStrategySet; private final FragmentationStrategyExecutorCreatorImpl fragmentationStrategyExecutorCreator; private final FragmentSequenceRepository fragmentSequenceRepository; + private final BucketisedMemberRepository bucketisedMemberRepository; public FragmentationStrategyCollectionImpl( - FragmentRepository fragmentRepository, - FragmentationStrategyExecutorCreatorImpl fragmentationStrategyExecutorCreator, - FragmentSequenceRepository fragmentSequenceRepository) { + FragmentRepository fragmentRepository, + FragmentationStrategyExecutorCreatorImpl fragmentationStrategyExecutorCreator, + FragmentSequenceRepository fragmentSequenceRepository, BucketisedMemberRepository bucketisedMemberRepository) { this.fragmentRepository = fragmentRepository; this.fragmentationStrategyExecutorCreator = fragmentationStrategyExecutorCreator; this.fragmentSequenceRepository = fragmentSequenceRepository; - this.fragmentationStrategySet = new HashSet<>(); + this.bucketisedMemberRepository = bucketisedMemberRepository; + this.fragmentationStrategySet = new HashSet<>(); } public List getFragmentationStrategyExecutors(String collectionName) { @@ -66,6 +69,7 @@ public void handleEventStreamDeletedEvent(EventStreamDeletedEvent event) { executor -> Objects.equals(executor.getViewName().getCollectionName(), event.collectionName())); fragmentRepository.deleteTreeNodesByCollection(event.collectionName()); fragmentSequenceRepository.deleteByCollection(event.collectionName()); + bucketisedMemberRepository.deleteByCollection(event.collectionName()); } @EventListener @@ -73,6 +77,7 @@ public void handleViewDeletedEvent(ViewDeletedEvent event) { removeFromStrategySet(executor -> Objects.equals(executor.getViewName(), event.getViewName())); fragmentRepository.removeLdesFragmentsOfView(event.getViewName().asString()); fragmentSequenceRepository.deleteByViewName(event.getViewName()); + bucketisedMemberRepository.deleteByViewName(event.getViewName()); } private void removeFromStrategySet(Predicate filterPredicate) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java index adc576bb5d..3dcc3b2f8a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java @@ -1,10 +1,13 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import io.micrometer.observation.Observation; -import org.apache.jena.rdf.model.Model; + +import java.util.List; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.GENERIC_TREE_RELATION; @@ -21,9 +24,9 @@ protected FragmentationStrategyDecorator(FragmentationStrategy fragmentationStra } @Override - public void addMemberToFragment(Fragment rootFragmentOfView, String memberId, Model memberModel, - Observation parentObservation) { - fragmentationStrategy.addMemberToFragment(rootFragmentOfView, memberId, memberModel, parentObservation); + public List addMemberToFragment(Fragment rootFragmentOfView, Member member, + Observation parentObservation) { + return fragmentationStrategy.addMemberToFragment(rootFragmentOfView, member, parentObservation); } protected void addRelationFromParentToChild(Fragment parentFragment, Fragment childFragment) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutor.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutor.java index 606743bc94..b994b682bf 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutor.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutor.java @@ -1,13 +1,14 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentSequenceRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services.MemberRetriever; import io.micrometer.observation.ObservationRegistry; -import org.apache.jena.rdf.model.Model; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; @@ -24,15 +25,18 @@ public class FragmentationStrategyExecutor { private final ObservationRegistry observationRegistry; private final MemberRetriever memberRetriever; private final FragmentSequenceRepository fragmentSequenceRepository; + private final BucketisedMemberSaver bucketisedMemberSaver; private boolean isExecutorActive = true; + @SuppressWarnings("java:S107") public FragmentationStrategyExecutor(ViewName viewName, FragmentationStrategy fragmentationStrategy, RootFragmentRetriever rootFragmentRetriever, ObservationRegistry observationRegistry, ExecutorService executorService, - MemberRetriever memberRetriever, - FragmentSequenceRepository fragmentSequenceRepository) { + MemberRetriever memberRetriever, + FragmentSequenceRepository fragmentSequenceRepository, + BucketisedMemberSaver bucketisedMemberSaver) { this.rootFragmentRetriever = rootFragmentRetriever; this.observationRegistry = observationRegistry; this.executorService = executorService; @@ -40,7 +44,8 @@ public FragmentationStrategyExecutor(ViewName viewName, this.viewName = viewName; this.memberRetriever = memberRetriever; this.fragmentSequenceRepository = fragmentSequenceRepository; - } + this.bucketisedMemberSaver = bucketisedMemberSaver; + } public void execute() { executorService.execute(addMembersToFragments()); @@ -72,9 +77,8 @@ private Optional getNextMemberToFragment(FragmentSequence lastProcessedS private FragmentSequence fragment(Member member) { var parentObservation = createNotStarted("execute fragmentation", observationRegistry).start(); var rootFragmentOfView = rootFragmentRetriever.retrieveRootFragmentOfView(viewName, parentObservation); - String memberId = member.id(); - Model memberModel = member.model(); - fragmentationStrategy.addMemberToFragment(rootFragmentOfView, memberId, memberModel, parentObservation); + List members = fragmentationStrategy.addMemberToFragment(rootFragmentOfView, member, parentObservation); + bucketisedMemberSaver.save(members); final FragmentSequence lastProcessedSequence = new FragmentSequence(viewName, member.sequenceNr()); fragmentSequenceRepository.saveLastProcessedSequence(lastProcessedSequence); parentObservation.stop(); diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java index d5a6b9d39f..c19dc53917 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java @@ -1,30 +1,17 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.MemberAllocatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import io.micrometer.observation.Observation; -import org.apache.jena.rdf.model.Model; -import org.springframework.context.ApplicationEventPublisher; -public class FragmentationStrategyImpl implements FragmentationStrategy { - private final FragmentRepository fragmentRepository; - private final ApplicationEventPublisher eventPublisher; +import java.util.List; - public FragmentationStrategyImpl(FragmentRepository fragmentRepository, - ApplicationEventPublisher eventPublisher) { - this.fragmentRepository = fragmentRepository; - this.eventPublisher = eventPublisher; - } +public class FragmentationStrategyImpl implements FragmentationStrategy { @Override - public void addMemberToFragment(Fragment fragment, String memberId, Model memberModel, - Observation parentObservation) { - eventPublisher.publishEvent( - new MemberAllocatedEvent(memberId, fragment.getViewName().getCollectionName(), - fragment.getViewName().getViewName(), fragment.getFragmentIdString())); - fragmentRepository.incrementNrOfMembersAdded(fragment.getFragmentId()); - + public List addMemberToFragment(Fragment fragment, Member member, + Observation parentObservation) { + return List.of(new BucketisedMember(member.id(), fragment.getViewName(), fragment.getFragmentIdString(), member.sequenceNr())); } - } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketisedMember.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketisedMember.java new file mode 100644 index 0000000000..2fe773762d --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketisedMember.java @@ -0,0 +1,6 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; + +public record BucketisedMember(String memberId, ViewName viewName, String fragmentId, Long sequenceNr) { +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyCreatorImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyCreatorImpl.java index 619a90c656..3b63c31034 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyCreatorImpl.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyCreatorImpl.java @@ -5,9 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyImpl; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Component; import java.util.List; @@ -16,28 +14,17 @@ public class FragmentationStrategyCreatorImpl implements FragmentationStrategyCreator { public static final String PAGINATION_FRAGMENTATION = "PaginationFragmentation"; private final ApplicationContext applicationContext; - private final FragmentRepository fragmentRepository; private final RootFragmentCreator rootFragmentCreator; - private final ApplicationEventPublisher eventPublisher; public FragmentationStrategyCreatorImpl(ApplicationContext applicationContext, - FragmentRepository fragmentRepository, - RootFragmentCreator rootFragmentCreator, - ApplicationEventPublisher eventPublisher) { + RootFragmentCreator rootFragmentCreator) { this.applicationContext = applicationContext; - this.fragmentRepository = fragmentRepository; this.rootFragmentCreator = rootFragmentCreator; - this.eventPublisher = eventPublisher; } public FragmentationStrategy createFragmentationStrategyForView(ViewSpecification viewSpecification) { rootFragmentCreator.createRootFragmentForView(viewSpecification.getName()); - FragmentationStrategy fragmentationStrategy = new FragmentationStrategyImpl(fragmentRepository, - eventPublisher); - FragmentationStrategyWrapper paginationWrapper = (FragmentationStrategyWrapper) applicationContext - .getBean(PAGINATION_FRAGMENTATION); - fragmentationStrategy = paginationWrapper.wrapFragmentationStrategy(applicationContext, fragmentationStrategy, - viewSpecification.getPaginationProperties()); + FragmentationStrategy fragmentationStrategy = new FragmentationStrategyImpl(); if (viewSpecification.getFragmentations() != null) { fragmentationStrategy = wrapFragmentationStrategy(viewSpecification.getFragmentations(), fragmentationStrategy); diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyExecutorCreatorImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyExecutorCreatorImpl.java index 8ed1a94245..8e8682a02b 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyExecutorCreatorImpl.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/FragmentationStrategyExecutorCreatorImpl.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.BucketisedMemberSaverImpl; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyExecutor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.RootFragmentRetriever; @@ -24,17 +25,20 @@ public class FragmentationStrategyExecutorCreatorImpl implements FragmentationSt private final MemberRetriever memberRetriever; private final FragmentSequenceRepository fragmentSequenceRepository; private final FragmentationStrategyCreator fragmentationStrategyCreator; + private final BucketisedMemberSaverImpl bucketisedMemberSaver; public FragmentationStrategyExecutorCreatorImpl(FragmentRepository fragmentRepository, ObservationRegistry observationRegistry, MemberRetriever memberRetriever, FragmentSequenceRepository fragmentSequenceRepository, - FragmentationStrategyCreator fragmentationStrategyCreator) { + FragmentationStrategyCreator fragmentationStrategyCreator, + BucketisedMemberSaverImpl bucketisedMemberSaver) { this.fragmentRepository = fragmentRepository; this.observationRegistry = observationRegistry; this.memberRetriever = memberRetriever; this.fragmentSequenceRepository = fragmentSequenceRepository; this.fragmentationStrategyCreator = fragmentationStrategyCreator; + this.bucketisedMemberSaver = bucketisedMemberSaver; } public FragmentationStrategyExecutor createExecutor(ViewName viewName, @@ -43,7 +47,7 @@ public FragmentationStrategyExecutor createExecutor(ViewName viewName, .createFragmentationStrategyForView(viewSpecification); final var rootFragmentRetriever = new RootFragmentRetriever(fragmentRepository, observationRegistry); return new FragmentationStrategyExecutor(viewName, fragmentationStrategy, rootFragmentRetriever, - observationRegistry, createExecutorService(), memberRetriever, fragmentSequenceRepository); + observationRegistry, createExecutorService(), memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); } /** diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java new file mode 100644 index 0000000000..8cd27f559c --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java @@ -0,0 +1,13 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; + +import java.util.List; + +public interface BucketisedMemberRepository { + void insertAll(List members); + List getFirstUnallocatedMember(ViewName viewName, Long sequenceNr); + void deleteByViewName(ViewName viewName); + void deleteByCollection(String collectionName); +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentSequenceRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentSequenceRepository.java index f3086e0b92..071aa624b0 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentSequenceRepository.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentSequenceRepository.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import java.util.Optional; diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImplTest.java index fcc3375f48..8b2b2c8114 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCollectionImplTest.java @@ -7,6 +7,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyExecutorCreatorImpl; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentSequenceRepository; import org.junit.jupiter.api.Test; @@ -25,10 +26,11 @@ class FragmentationStrategyCollectionImplTest { FragmentationStrategyExecutorCreatorImpl.class); private final FragmentRepository fragmentRepository = mock(FragmentRepository.class); private final FragmentSequenceRepository fragmentSequenceRepository = mock(FragmentSequenceRepository.class); + private final BucketisedMemberRepository bucketisedMemberRepository = mock(BucketisedMemberRepository.class); private final FragmentationStrategyCollectionImpl fragmentationStrategyCollection = new FragmentationStrategyCollectionImpl( - fragmentRepository, - fragmentationStrategyExecutorCreator, fragmentSequenceRepository); + fragmentRepository, fragmentationStrategyExecutorCreator, + fragmentSequenceRepository, bucketisedMemberRepository); @Test void when_ViewAddedEventIsReceived_FragmentationStrategyIsAddedToMap() { @@ -81,11 +83,13 @@ void when_ViewDeletedEventIsReceived_FragmentationStrategyIsRemovedFromMap() { assertTrue(fragmentationStrategyCollection.getFragmentationStrategyExecutors(COLLECTION_NAME).isEmpty()); // verify that the executor is shutdown before removing everything from repos. var fragmentationStrategyExecutor = initResult.fragmentationStrategyExecutor; - InOrder inOrder = inOrder(fragmentRepository, fragmentSequenceRepository, fragmentationStrategyExecutor); + InOrder inOrder = inOrder(fragmentRepository, fragmentSequenceRepository, + fragmentationStrategyExecutor, bucketisedMemberRepository); inOrder.verify(fragmentationStrategyExecutor).shutdown(); inOrder.verify(fragmentRepository) .removeLdesFragmentsOfView(initResult.viewSpecification().getName().asString()); inOrder.verify(fragmentSequenceRepository).deleteByViewName(initResult.viewName); + inOrder.verify(bucketisedMemberRepository).deleteByViewName(initResult.viewName); } @Test @@ -112,10 +116,12 @@ void should_DeleteTreeNodesByCollection_when_EventStreamDeletedEventIsReceived() assertTrue(fragmentationStrategyCollection.getFragmentationStrategyExecutors(COLLECTION_NAME).isEmpty()); // verify that the executor is shutdown before removing everything from repos. var fragmentationStrategyExecutor = initResult.fragmentationStrategyExecutor; - InOrder inOrder = inOrder(fragmentRepository, fragmentSequenceRepository, fragmentationStrategyExecutor); + InOrder inOrder = inOrder(fragmentRepository, fragmentSequenceRepository, + fragmentationStrategyExecutor, bucketisedMemberRepository); inOrder.verify(fragmentationStrategyExecutor).shutdown(); inOrder.verify(fragmentRepository).deleteTreeNodesByCollection(initResult.viewName.getCollectionName()); inOrder.verify(fragmentSequenceRepository).deleteByCollection(initResult.viewName.getCollectionName()); + inOrder.verify(bucketisedMemberRepository).deleteByCollection(initResult.viewName.getCollectionName()); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java index ca7e971320..ca0d8d4bb0 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java @@ -6,18 +6,16 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyCreatorImpl; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.RootFragmentCreator; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.Mockito; import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationEventPublisher; import java.util.List; import java.util.Map; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyCreatorImpl.PAGINATION_FRAGMENTATION; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.*; @@ -35,54 +33,34 @@ class FragmentationStrategyCreatorImplTest { @BeforeEach void setUp() { applicationContext = mock(ApplicationContext.class); - FragmentRepository fragmentRepository = mock(FragmentRepository.class); rootFragmentCreator = mock(RootFragmentCreator.class); - ApplicationEventPublisher eventPublisher = mock(ApplicationEventPublisher.class); fragmentationStrategyCreator = new FragmentationStrategyCreatorImpl( - applicationContext, fragmentRepository, rootFragmentCreator, - eventPublisher); + applicationContext, rootFragmentCreator); } @Test void when_ViewSpecificationFragmentationConfigIsNull_FragmentationStrategyImplIsReturned() { ViewSpecification viewSpecification = new ViewSpecification(VIEW_NAME, List.of(), List.of(), 100); - FragmentationStrategyWrapper paginationFragmentationStrategyWrapper = Mockito - .mock(FragmentationStrategyWrapper.class); - when(applicationContext.getBean(PAGINATION_FRAGMENTATION)).thenReturn(paginationFragmentationStrategyWrapper); - FragmentationStrategy paginationFragmentationStrategy = Mockito.mock(FragmentationStrategy.class); - when(paginationFragmentationStrategyWrapper.wrapFragmentationStrategy(eq(applicationContext), - any(FragmentationStrategyImpl.class), - eq(viewSpecification.getPaginationProperties()))) - .thenReturn(paginationFragmentationStrategy); FragmentationStrategy fragmentationStrategy = fragmentationStrategyCreator .createFragmentationStrategyForView(viewSpecification); - assertEquals(paginationFragmentationStrategy, fragmentationStrategy); + assertThat(fragmentationStrategy).isOfAnyClassIn(FragmentationStrategyImpl.class); InOrder inOrder = inOrder(applicationContext, rootFragmentCreator); inOrder.verify(rootFragmentCreator).createRootFragmentForView(viewSpecification.getName()); - inOrder.verify(applicationContext).getBean(PAGINATION_FRAGMENTATION); inOrder.verifyNoMoreInteractions(); } @Test void when_ViewSpecificationFragmentationConfigIsNotNull_WrappedFragmentationStrategyIsReturned() { ViewSpecification viewSpecification = getViewSpecification(); - FragmentationStrategyWrapper paginationFragmentationStrategyWrapper = Mockito - .mock(FragmentationStrategyWrapper.class); - when(applicationContext.getBean(PAGINATION_FRAGMENTATION)).thenReturn(paginationFragmentationStrategyWrapper); - FragmentationStrategy paginationFragmentationStrategy = Mockito.mock(FragmentationStrategy.class); - when(paginationFragmentationStrategyWrapper.wrapFragmentationStrategy(eq(applicationContext), - any(FragmentationStrategyImpl.class), - eq(viewSpecification.getPaginationProperties()))) - .thenReturn(paginationFragmentationStrategy); FragmentationStrategyWrapper timebasedFragmentationStrategyWrapper = Mockito .mock(FragmentationStrategyWrapper.class); when(applicationContext.getBean(TIMEBASED)).thenReturn(timebasedFragmentationStrategyWrapper); FragmentationStrategy timebasedFragmentationStrategy = Mockito.mock(FragmentationStrategy.class); - when(timebasedFragmentationStrategyWrapper.wrapFragmentationStrategy(applicationContext, - paginationFragmentationStrategy, - new ConfigProperties(TIMEBASED_PROPERTIES))) + when(timebasedFragmentationStrategyWrapper.wrapFragmentationStrategy(eq(applicationContext), + any(FragmentationStrategyImpl.class), + eq(new ConfigProperties(TIMEBASED_PROPERTIES)))) .thenReturn(timebasedFragmentationStrategy); FragmentationStrategyWrapper geospatialFragmentationStrategyWrapper = Mockito @@ -99,7 +77,6 @@ timebasedFragmentationStrategy, new ConfigProperties(GEOSPATIAL_PROPERTIES))) assertEquals(geospatialFragmentationStrategy, fragmentationStrategy); InOrder inOrder = inOrder(applicationContext, rootFragmentCreator); inOrder.verify(rootFragmentCreator).createRootFragmentForView(viewSpecification.getName()); - inOrder.verify(applicationContext).getBean(PAGINATION_FRAGMENTATION); inOrder.verify(applicationContext).getBean(TIMEBASED); inOrder.verify(applicationContext).getBean(GEOSPATIAL); inOrder.verifyNoMoreInteractions(); diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java index 3f9686a548..3f6e80a6ff 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java @@ -53,11 +53,10 @@ void when_DecoratorAddsMemberToFragment_WrappedFragmentationStrategyIsCalled() { Fragment parentFragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of())); Member member = mock(Member.class); Observation span = mock(Observation.class); - fragmentationStrategyDecorator.addMemberToFragment(parentFragment, member.id(), member.model(), + fragmentationStrategyDecorator.addMemberToFragment(parentFragment, member, span); verify(fragmentationStrategy, - Mockito.times(1)).addMemberToFragment(parentFragment, member.id(), - member.model(), span); + Mockito.times(1)).addMemberToFragment(parentFragment, member, span); } static class FragmentationStrategyDecoratorTestImpl extends diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutorTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutorTest.java index 3914bf1953..2435c91f85 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyExecutorTest.java @@ -3,7 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentSequenceRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services.MemberRetriever; @@ -47,6 +47,8 @@ class FragmentationStrategyExecutorTest { private MemberRetriever memberRetriever; @Mock private FragmentSequenceRepository fragmentSequenceRepository; + @Mock + private BucketisedMemberSaverImpl bucketisedMemberSaver; @Nested class ExecuteNext { @@ -57,20 +59,20 @@ class ExecuteNext { void when_ExecuteIsCalled_then_AllLogicIsWrappedByTheExecutorService() { ObservationRegistry observationRegistry = mock(ObservationRegistry.class); var executor = new FragmentationStrategyExecutor(viewName, fragmentationStrategy, rootFragmentRetriever, - observationRegistry, executorService, memberRetriever, fragmentSequenceRepository); + observationRegistry, executorService, memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); executor.execute(); verify(executorService).execute(any()); verifyNoMoreInteractions(fragmentationStrategy, rootFragmentRetriever, observationRegistry, - memberRetriever, fragmentSequenceRepository); + memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); } @Test void when_ExecuteIsCalled_then_TheExecutorService_should_NotFragmentAnythingIfNotPresent() { var executor = new FragmentationStrategyExecutor(viewName, fragmentationStrategy, rootFragmentRetriever, ObservationRegistry.create(), - MoreExecutors.newDirectExecutorService(), memberRetriever, fragmentSequenceRepository); + MoreExecutors.newDirectExecutorService(), memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); FragmentSequence fragmentSequence = new FragmentSequence(viewName, 0L); when(fragmentSequenceRepository.findLastProcessedSequence(viewName)) .thenReturn(Optional.of(fragmentSequence)); @@ -88,20 +90,21 @@ void when_ExecuteIsCalled_then_TheExecutorService_should_FragmentTheNextMemberIf ObservationRegistry observationRegistry = ObservationRegistry.create(); var executor = new FragmentationStrategyExecutor(viewName, fragmentationStrategy, rootFragmentRetriever, observationRegistry, MoreExecutors.newDirectExecutorService(), memberRetriever, - fragmentSequenceRepository); + fragmentSequenceRepository, bucketisedMemberSaver); when(fragmentSequenceRepository.findLastProcessedSequence(viewName)).thenReturn(Optional.empty()); String memberId = "id"; Model memberModel = ModelFactory.createDefaultModel(); long sequenceNr = 1L; + Member member = new Member(memberId, memberModel, sequenceNr); when(memberRetriever.findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(viewName.getCollectionName(), FragmentSequence.createNeverProcessedSequence(viewName).sequenceNr())) - .thenReturn(Optional.of(new Member(memberId, memberModel, sequenceNr))); + .thenReturn(Optional.of(member)); final Fragment rootFragment = new Fragment(new LdesFragmentIdentifier(viewName, List.of())); when(rootFragmentRetriever.retrieveRootFragmentOfView(eq(viewName), any())).thenReturn(rootFragment); executor.execute(); - verify(fragmentationStrategy).addMemberToFragment(eq(rootFragment), eq(memberId), eq(memberModel), any()); + verify(fragmentationStrategy).addMemberToFragment(eq(rootFragment), eq(member), any()); verify(fragmentSequenceRepository).saveLastProcessedSequence(new FragmentSequence(viewName, sequenceNr)); } } @@ -110,7 +113,7 @@ void when_ExecuteIsCalled_then_TheExecutorService_should_FragmentTheNextMemberIf void isPartOfCollection() { final ViewName viewNameA = ViewName.fromString("col/viewA"); final FragmentationStrategyExecutor executorA = new FragmentationStrategyExecutor(viewNameA, null, null, null, - null, memberRetriever, fragmentSequenceRepository); + null, memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); assertTrue(executorA.isPartOfCollection(viewNameA.getCollectionName())); assertFalse(executorA.isPartOfCollection("other")); @@ -120,7 +123,7 @@ void isPartOfCollection() { void shutDown() throws InterruptedException { final ViewName viewName = ViewName.fromString("col/viewA"); final FragmentationStrategyExecutor executor = new FragmentationStrategyExecutor(viewName, null, null, null, - executorService, memberRetriever, fragmentSequenceRepository); + executorService, memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); executor.shutdown(); @@ -132,7 +135,7 @@ void shutDown() throws InterruptedException { void getViewName() { final ViewName viewNameA = ViewName.fromString("col/viewA"); final FragmentationStrategyExecutor executorA = new FragmentationStrategyExecutor(viewNameA, null, null, null, - null, memberRetriever, fragmentSequenceRepository); + null, memberRetriever, fragmentSequenceRepository, bucketisedMemberSaver); assertEquals(viewNameA, executorA.getViewName()); } @@ -152,22 +155,23 @@ static class EqualityTestProvider implements ArgumentsProvider { private static final ViewName viewNameA = ViewName.fromString("col/viewA"); private static final FragmentationStrategyExecutor executorA = new FragmentationStrategyExecutor(viewNameA, - null, null, null, null, null, null); + null, null, null, null, null, null, null); @Override public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of(equals(), executorA, executorA), Arguments.of(equals(), executorA, - new FragmentationStrategyExecutor(viewNameA, null, null, null, null, null, null)), + new FragmentationStrategyExecutor(viewNameA, null, null, null, null, null, null, null)), Arguments.of(equals(), executorA, new FragmentationStrategyExecutor(viewNameA, mock(FragmentationStrategy.class), mock(RootFragmentRetriever.class), mock(ObservationRegistry.class), mock(ExecutorService.class), mock(MemberRetriever.class), - mock(FragmentSequenceRepository.class))), + mock(FragmentSequenceRepository.class), + mock(BucketisedMemberSaverImpl.class))), Arguments.of(notEquals(), executorA, new FragmentationStrategyExecutor(ViewName.fromString("col/viewB"), null, null, null, - null, null, null))); + null, null, null, null))); } private static BiConsumer equals() { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java index 378cbb386f..1a40b77767 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java @@ -2,33 +2,38 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import io.micrometer.observation.Observation; import org.junit.jupiter.api.Test; -import org.springframework.context.ApplicationEventPublisher; import java.util.List; -import static org.mockito.Mockito.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class FragmentationStrategyImplTest { - private final FragmentRepository fragmentRepository = mock(FragmentRepository.class); - private final ApplicationEventPublisher eventPublisher = mock(ApplicationEventPublisher.class); - - private final FragmentationStrategyImpl fragmentationStrategy = new FragmentationStrategyImpl( - fragmentRepository, - eventPublisher); + private static final String MEMBER_ID = "memberId"; + private static final long SEQ_NR = 5L; + private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); + private static final LdesFragmentIdentifier FRAGMENT_ID = new LdesFragmentIdentifier(VIEW_NAME, List.of()); + private final FragmentationStrategyImpl fragmentationStrategy = new FragmentationStrategyImpl(); @Test void when_memberIsAddedToFragment_FragmentationStrategyImplSavesUpdatedFragment() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "view"), - List.of())); + Fragment fragment = new Fragment(FRAGMENT_ID); Member member = mock(Member.class); - when(member.id()).thenReturn("memberId"); + when(member.id()).thenReturn(MEMBER_ID); + when(member.sequenceNr()).thenReturn(SEQ_NR); - fragmentationStrategy.addMemberToFragment(fragment, member.id(), member.model(), any()); + List members = fragmentationStrategy.addMemberToFragment(fragment, member, mock(Observation.class)); - verify(fragmentRepository, times(1)).incrementNrOfMembersAdded(fragment.getFragmentId()); + assertThat(members).hasSize(1); + assertThat(members.getFirst()).hasFieldOrPropertyWithValue("memberId", MEMBER_ID) + .hasFieldOrPropertyWithValue("viewName", VIEW_NAME) + .hasFieldOrPropertyWithValue("fragmentId", FRAGMENT_ID.asDecodedFragmentId()) + .hasFieldOrPropertyWithValue("sequenceNr", SEQ_NR); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java index 8d9096a93a..6ebd67a798 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import org.junit.jupiter.api.Test; diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java index ffc4864e29..936cb5ab10 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java @@ -2,14 +2,16 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.bucketising.GeospatialBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.fragments.GeospatialFragmentCreator; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import org.apache.jena.rdf.model.Model; +import java.util.Collection; import java.util.List; import java.util.Set; @@ -34,15 +36,15 @@ public GeospatialFragmentationStrategy(FragmentationStrategy fragmentationStrate } @Override - public void addMemberToFragment(Fragment parentFragment, String memberId, Model memberModel, - Observation parentObservation) { + public List addMemberToFragment(Fragment parentFragment, Member member, + Observation parentObservation) { Observation geospatialFragmentationObservation = Observation.createNotStarted("geospatial fragmentation", observationRegistry) .parentObservation(parentObservation) .start(); getRootTileFragment(parentFragment); - Set tiles = geospatialBucketiser.bucketise(memberId, memberModel); + Set tiles = geospatialBucketiser.bucketise(member.id(), member.model()); List fragments = tiles .stream() @@ -54,11 +56,14 @@ public void addMemberToFragment(Fragment parentFragment, String memberId, Model } }).toList(); - fragments + List members = fragments .parallelStream() - .forEach(ldesFragment -> super.addMemberToFragment(ldesFragment, memberId, memberModel, - geospatialFragmentationObservation)); + .map(ldesFragment -> super.addMemberToFragment(ldesFragment, member, + geospatialFragmentationObservation)) + .flatMap(Collection::stream) + .toList(); geospatialFragmentationObservation.stop(); + return members; } private void getRootTileFragment(Fragment parentFragment) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java index 60ca15d58a..d0f85c8332 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java @@ -59,18 +59,17 @@ void when_MemberIsAddedToFragment_GeospatialFragmentationIsApplied() { Fragment tileFragmentTwo = mockCreationGeospatialFragment("2/2/2"); Fragment tileFragmentThree = mockCreationGeospatialFragment("3/3/3"); - geospatialFragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member.id(), - member.model(), mock(Observation.class)); + geospatialFragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member, mock(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(tileFragmentOne), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(tileFragmentTwo), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(tileFragmentThree), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verifyNoMoreInteractions(decoratedFragmentationStrategy); } @Test @@ -83,12 +82,11 @@ void when_MemberIsAddedToDefaultFragment_GeospatialFragmentationIsApplied() { PARENT_FRAGMENT)) .thenReturn(defaultTileFragment); - geospatialFragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member.id(), - member.model(), mock(Observation.class)); + geospatialFragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member, mock(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(defaultTileFragment), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verifyNoMoreInteractions(decoratedFragmentationStrategy); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategy.java deleted file mode 100644 index 6989bd8414..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategy.java +++ /dev/null @@ -1,46 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services.OpenPageProvider; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.jena.rdf.model.Model; - -import static java.lang.Boolean.TRUE; - -public class PaginationStrategy extends FragmentationStrategyDecorator { - - public static final String PAGINATION_FRAGMENTATION = "PaginationFragmentation"; - - private final OpenPageProvider openPageProvider; - - private final ObservationRegistry observationRegistry; - - public PaginationStrategy(FragmentationStrategy fragmentationStrategy, - OpenPageProvider openPageProvider, ObservationRegistry observationRegistry, - FragmentRepository fragmentRepository) { - super(fragmentationStrategy, fragmentRepository); - this.openPageProvider = openPageProvider; - this.observationRegistry = observationRegistry; - } - - @Override - public void addMemberToFragment(Fragment parentFragment, String memberId, Model memberModel, - Observation parentObservation) { - Observation paginationObservation = Observation.createNotStarted(PAGINATION_FRAGMENTATION, - observationRegistry) - .parentObservation(parentObservation) - .start(); - ImmutablePair ldesFragment = openPageProvider - .retrieveOpenFragmentOrCreateNewFragment(parentFragment); - if (TRUE.equals(ldesFragment.getRight())) { - super.addRelationFromParentToChild(parentFragment, ldesFragment.getLeft()); - } - super.addMemberToFragment(ldesFragment.getLeft(), memberId, memberModel, paginationObservation); - paginationObservation.stop(); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyAutoConfiguration.java b/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyAutoConfiguration.java deleted file mode 100644 index 316304c138..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyAutoConfiguration.java +++ /dev/null @@ -1,20 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.PaginationStrategy.PAGINATION_FRAGMENTATION; - -@Configuration -@EnableConfigurationProperties() -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes.server") -public class PaginationStrategyAutoConfiguration { - - @SuppressWarnings("java:S6830") - @Bean(PAGINATION_FRAGMENTATION) - public PaginationStrategyWrapper paginationStrategyWrapper() { - return new PaginationStrategyWrapper(); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapper.java deleted file mode 100644 index ce34198c87..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapper.java +++ /dev/null @@ -1,48 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.config.PaginationConfig; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services.OpenPageProvider; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services.PageCreator; -import io.micrometer.observation.ObservationRegistry; -import org.springframework.context.ApplicationContext; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.config.PaginationProperties.BIDIRECTIONAL_RELATIONS; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.config.PaginationProperties.MEMBER_LIMIT; - -public class PaginationStrategyWrapper implements FragmentationStrategyWrapper { - - public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, - FragmentationStrategy fragmentationStrategy, ConfigProperties fragmentationProperties) { - FragmentRepository fragmentRepository = applicationContext.getBean(FragmentRepository.class); - ObservationRegistry observationRegistry = applicationContext.getBean(ObservationRegistry.class); - - OpenPageProvider openFragmentProvider = getOpenPageProvider(fragmentationProperties, - fragmentRepository); - return new PaginationStrategy(fragmentationStrategy, - openFragmentProvider, observationRegistry, fragmentRepository); - - } - - private OpenPageProvider getOpenPageProvider(ConfigProperties properties, - FragmentRepository fragmentRepository) { - PaginationConfig paginationConfig = createPaginationConfig(properties); - PageCreator pageFragmentCreator = getPageCreator( - fragmentRepository, paginationConfig.bidirectionalRelations()); - return new OpenPageProvider(pageFragmentCreator, fragmentRepository, - paginationConfig.memberLimit()); - } - - private PageCreator getPageCreator(FragmentRepository fragmentRepository, boolean bidirectionalRelations) { - return new PageCreator( - fragmentRepository, bidirectionalRelations); - } - - private PaginationConfig createPaginationConfig(ConfigProperties properties) { - return new PaginationConfig(Long.valueOf(properties.get(MEMBER_LIMIT)), - Boolean.parseBoolean(properties.getOrDefault(BIDIRECTIONAL_RELATIONS, "true"))); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationConfig.java b/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationConfig.java deleted file mode 100644 index 8738519f10..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationConfig.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.config; - -public record PaginationConfig(Long memberLimit, boolean bidirectionalRelations) { -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyTest.java deleted file mode 100644 index 04569aece4..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyTest.java +++ /dev/null @@ -1,86 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services.OpenPageProvider; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.InOrder; - -import java.util.List; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants.PaginationConstants.PAGE_NUMBER; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.*; - -class PaginationStrategyTest { - - private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private final OpenPageProvider openPageProvider = mock(OpenPageProvider.class); - private final FragmentationStrategy decoratedFragmentationStrategy = mock(FragmentationStrategy.class); - private FragmentationStrategy fragmentationStrategy; - private static Fragment PARENT_FRAGMENT; - private static Fragment OPEN_FRAGMENT; - private final FragmentRepository fragmentRepository = mock(FragmentRepository.class); - - @BeforeEach - void setUp() { - PARENT_FRAGMENT = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of())); - OPEN_FRAGMENT = PARENT_FRAGMENT.createChild(new FragmentPair(PAGE_NUMBER, "1")); - fragmentationStrategy = new PaginationStrategy(decoratedFragmentationStrategy, - openPageProvider, ObservationRegistry.create(), - fragmentRepository); - } - - @Test - void when_MemberIsAddedToFragment_PaginationIsApplied() { - Member member = mock(Member.class); - when(member.id()).thenReturn("memberId"); - when(openPageProvider.retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT)) - .thenReturn(new ImmutablePair<>(OPEN_FRAGMENT, false)); - - fragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, - member.id(), member.model(), any(Observation.class)); - - InOrder inOrder = inOrder(openPageProvider, fragmentRepository, - decoratedFragmentationStrategy); - inOrder.verify(openPageProvider, - times(1)).retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT); - inOrder.verify(decoratedFragmentationStrategy, - times(1)) - .addMemberToFragment(eq(OPEN_FRAGMENT), any(), any(), any(Observation.class)); - inOrder.verifyNoMoreInteractions(); - } - - @Test - void when_MemberIsAddedToFirstFragment_PaginationIsAppliedAndRelationIsAdded() { - Member member = mock(Member.class); - when(member.id()).thenReturn("memberId"); - when(openPageProvider.retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT)) - .thenReturn(new ImmutablePair<>(OPEN_FRAGMENT, true)); - - fragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, - member.id(), member.model(), any(Observation.class)); - - InOrder inOrder = inOrder(openPageProvider, fragmentRepository, - decoratedFragmentationStrategy); - inOrder.verify(openPageProvider, - times(1)).retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT); - inOrder.verify(fragmentRepository, - times(1)).saveFragment(PARENT_FRAGMENT); - inOrder.verify(decoratedFragmentationStrategy, - times(1)) - .addMemberToFragment(eq(OPEN_FRAGMENT), any(), any(), any(Observation.class)); - - inOrder.verifyNoMoreInteractions(); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapperTest.java b/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapperTest.java deleted file mode 100644 index 8ab685a0ea..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/PaginationStrategyWrapperTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.context.ApplicationContext; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -class PaginationStrategyWrapperTest { - private final ApplicationContext applicationContext = mock(ApplicationContext.class); - private final FragmentationStrategy fragmentationStrategy = mock(FragmentationStrategy.class); - private PaginationStrategyWrapper paginationUpdater; - - @BeforeEach - void setUp() { - paginationUpdater = new PaginationStrategyWrapper(); - } - - @Test - void when_FragmentationStrategyIsUpdated_TimebasedFragmentationStrategyIsReturned() { - ConfigProperties properties = new ConfigProperties(Map.of("memberLimit", "5")); - FragmentationStrategy decoratedFragmentationStrategy = paginationUpdater - .wrapFragmentationStrategy(applicationContext, fragmentationStrategy, properties); - assertTrue(decoratedFragmentationStrategy instanceof PaginationStrategy); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java index 1048de910c..c382df6faf 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java @@ -2,13 +2,17 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.bucketising.ReferenceBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceFragmentCreator; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import org.apache.jena.rdf.model.Model; + +import java.util.Collection; +import java.util.List; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceFragmentCreator.FRAGMENT_KEY_REFERENCE_ROOT; @@ -32,20 +36,23 @@ public ReferenceFragmentationStrategy(FragmentationStrategy fragmentationStrateg } @Override - public void addMemberToFragment(Fragment parentFragment, String memberId, Model memberModel, - Observation parentObservation) { + public List addMemberToFragment(Fragment parentFragment, Member member, + Observation parentObservation) { final var fragmentationObservation = startObservation(parentObservation); final var rootFragment = getOrCreateRootFragment(parentFragment); var fragments = referenceBucketiser - .bucketise(memberId, memberModel) + .bucketise(member.id(), member.model()) .stream() .map(reference -> fragmentCreator.getOrCreateFragment(parentFragment, reference, rootFragment)) .toList(); - fragments.parallelStream() - .forEach(ldesFragment -> super.addMemberToFragment(ldesFragment, memberId, memberModel, fragmentationObservation)); + List members = fragments.parallelStream() + .map(ldesFragment -> super.addMemberToFragment(ldesFragment, member, fragmentationObservation)) + .flatMap(Collection::stream) + .toList(); fragmentationObservation.stop(); + return members; } private Observation startObservation(Observation parentObservation) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java index 695fa59fe2..16924a8ea4 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java @@ -63,17 +63,17 @@ void when_MemberIsAddedToFragment_ThenReferenceFragmentationIsApplied() { Fragment referenceFragmentThree = mockCreationReferenceFragment(typeAdres); referenceFragmentationStrategy - .addMemberToFragment(PARENT_FRAGMENT, member.id(), member.model(), mock(Observation.class)); + .addMemberToFragment(PARENT_FRAGMENT, member, mock(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(referenceFragmentOne), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(referenceFragmentTwo), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(referenceFragmentThree), - any(), any(), any(Observation.class)); + any(), any(Observation.class)); verifyNoMoreInteractions(decoratedFragmentationStrategy); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java index 66612bc8a4..5c62f9973b 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java @@ -3,7 +3,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.LocalDateTimeConverter; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; @@ -16,6 +18,7 @@ import org.slf4j.LoggerFactory; import java.time.LocalDateTime; +import java.util.List; import java.util.Optional; public class HierarchicalTimeBasedFragmentationStrategy extends FragmentationStrategyDecorator { @@ -39,16 +42,17 @@ public HierarchicalTimeBasedFragmentationStrategy(FragmentationStrategy fragment } @Override - public void addMemberToFragment(Fragment parentFragment, String memberId, Model memberModel, - Observation parentObservation) { + public List addMemberToFragment(Fragment parentFragment, Member member, + Observation parentObservation) { final Observation fragmentationObservation = startFragmentationObservation(parentObservation); - Fragment fragment = getFragmentationTimestamp(memberId, memberModel) + Fragment fragment = getFragmentationTimestamp(member.id(), member.model()) .map(timestamp -> fragmentFinder.getLowestFragment(parentFragment, timestamp, Granularity.YEAR)) .orElseGet(() -> fragmentFinder.getDefaultFragment(parentFragment)); - super.addMemberToFragment(fragment, memberId, memberModel, fragmentationObservation); + List members = super.addMemberToFragment(fragment, member, fragmentationObservation); fragmentationObservation.stop(); + return members; } private Optional getFragmentationTimestamp(String memberId, Model memberModel) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java index feb78aec3f..043d830866 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java @@ -60,14 +60,14 @@ void when_FragmentationCalled_Then_FunctionsAreCalled() { when(fragmentFinder.getLowestFragment(PARENT_FRAGMENT, fragmentationTimestamp, Granularity.YEAR)) .thenReturn(CHILD_FRAGMENT); - fragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member.id(), member.model(), + fragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member, mock(Observation.class)); InOrder inOrder = Mockito.inOrder(fragmentFinder, decoratedFragmentationStrategy); inOrder.verify(fragmentFinder).getLowestFragment(PARENT_FRAGMENT, fragmentationTimestamp, Granularity.YEAR); inOrder.verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(CHILD_FRAGMENT), any(), - any(), any(Observation.class)); + any(Observation.class)); } @Test @@ -76,14 +76,14 @@ void when_FragmentationCalledForMemberWithMissingTimestamp_Then_FunctionsAreCall when(fragmentFinder.getDefaultFragment(PARENT_FRAGMENT)) .thenReturn(CHILD_FRAGMENT); - fragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member.id(), member.model(), + fragmentationStrategy.addMemberToFragment(PARENT_FRAGMENT, member, mock(Observation.class)); InOrder inOrder = Mockito.inOrder(fragmentFinder, decoratedFragmentationStrategy); inOrder.verify(fragmentFinder).getDefaultFragment(PARENT_FRAGMENT); inOrder.verify(decoratedFragmentationStrategy, times(1)).addMemberToFragment(eq(CHILD_FRAGMENT), any(), - any(), any(Observation.class)); + any(Observation.class)); } } diff --git a/ldes-fragmentisers/pom.xml b/ldes-fragmentisers/pom.xml index b0cc9ab2bf..339ce59521 100644 --- a/ldes-fragmentisers/pom.xml +++ b/ldes-fragmentisers/pom.xml @@ -15,7 +15,6 @@ ldes-fragmentisers-common ldes-fragmentisers-geospatial ldes-fragmentisers-timebased-hierarchical - ldes-fragmentisers-pagination ldes-fragmentisers-reference diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index 7142b2b5fb..ac19a0afc2 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -146,11 +146,11 @@ - fragmentation-pagination + fragmentation-reference be.vlaanderen.informatievlaanderen.vsds - ldes-fragmentisers-pagination + ldes-fragmentisers-reference ${project.version} @@ -161,16 +161,16 @@ - fragmentation-reference + pagination be.vlaanderen.informatievlaanderen.vsds - ldes-fragmentisers-reference + ldes-server-pagination ${project.version} be.vlaanderen.informatievlaanderen.vsds - postgres-fragmentation-repository + postgres-pagination-repository ${project.version} @@ -225,15 +225,5 @@ - - mongo-to-postgres-migration - - - be.vlaanderen.informatievlaanderen.vsds - ldes-server-infra-migration - ${project.version} - - - diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java b/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java index f8e0f6f630..fdb7105c0f 100644 --- a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java +++ b/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java @@ -13,8 +13,6 @@ public class FragmentationConfigCompaction { @Bean("compactionFragmentation") public FragmentationStrategy compactionFragmentationStrategy(FragmentRepository fragmentRepository, ApplicationEventPublisher eventPublisher) { - return new FragmentationStrategyImpl(fragmentRepository, - eventPublisher); - + return new FragmentationStrategyImpl(); } } diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/CompactionIntegrationTest.java b/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/CompactionIntegrationTest.java index b44753f087..4bb6ef3313 100644 --- a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/CompactionIntegrationTest.java +++ b/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/CompactionIntegrationTest.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.SchedulingConfigCompaction; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.BulkMemberAllocatedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.AllocationRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyImpl; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.retention.spi.RetentionPolicyEmptinessChecker; import io.cucumber.spring.CucumberContextConfiguration; @@ -45,8 +44,6 @@ public class CompactionIntegrationTest { @Autowired @MockBean AllocationRepository allocationRepository; - @Autowired - FragmentationStrategyImpl fragmentationStrategy; @TestComponent protected static class EventConsumer { @@ -58,11 +55,6 @@ public void consumeEvent(BulkMemberAllocatedEvent testEvent) { @TestConfiguration public static class CompactionIntegrationTestConfiguration { - @Bean("compactionFragmentation") - public FragmentationStrategyImpl fragmentationStrategy() { - return mock(FragmentationStrategyImpl.class); - } - @Bean public RetentionPolicyEmptinessChecker retentionPolicyEmptinessChecker() { return mock(RetentionPolicyEmptinessChecker.class); diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberBucketisedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberBucketisedEvent.java new file mode 100644 index 0000000000..16d4b7fe6e --- /dev/null +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberBucketisedEvent.java @@ -0,0 +1,6 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; + +public record MemberBucketisedEvent(ViewName viewName) { +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequence.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java similarity index 73% rename from ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequence.java rename to ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java index b5a1d2c9a2..c3d02660fa 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequence.java +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java @@ -1,6 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; public record FragmentSequence(ViewName viewName, long sequenceNr) { diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 5e97ff2787..67fa480e56 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -15,6 +15,8 @@ postgres-fragmentation-repository postgres-ingest-repository postgres-retention-repository + postgres-liquibase + postgres-pagination-repository ldes-server-infra-postgres diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketisedMemberPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketisedMemberPostgresRepository.java new file mode 100644 index 0000000000..1386b5ea46 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketisedMemberPostgresRepository.java @@ -0,0 +1,46 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper.MemberBucketEntityMapper; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.MemberBucketEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Component +public class BucketisedMemberPostgresRepository implements BucketisedMemberRepository { + private static final String COLLECTION_VIEW_SEPARATOR = "/"; + private final MemberBucketEntityRepository memberBucketEntityRepository; + private final MemberBucketEntityMapper mapper; + + public BucketisedMemberPostgresRepository(MemberBucketEntityRepository memberBucketEntityRepository, MemberBucketEntityMapper mapper) { + this.memberBucketEntityRepository = memberBucketEntityRepository; + this.mapper = mapper; + } + + @Override + public void insertAll(List members) { + memberBucketEntityRepository.saveAll(members.stream().map(mapper::toMemberBucketisationEntity).toList()); + } + + @Override + public List getFirstUnallocatedMember(ViewName viewName, Long sequenceNr) { + return memberBucketEntityRepository.findAllByViewNameAndSequenceNr(viewName.asString(), sequenceNr) + .stream().map(mapper::toBucketisedMember).toList(); + } + + @Override + @Transactional + public void deleteByViewName(ViewName viewName) { + memberBucketEntityRepository.deleteAllByViewName(viewName.asString()); + } + + @Override + @Transactional + public void deleteByCollection(String collectionName) { + memberBucketEntityRepository.deleteAllByViewNameStartingWith(collectionName + COLLECTION_VIEW_SEPARATOR); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentSequencePostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentSequencePostgresRepository.java index 0f36cfa056..fb816d3223 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentSequencePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentSequencePostgresRepository.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper.SequenceEntityMapper; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.SequenceEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentSequenceRepository; diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java new file mode 100644 index 0000000000..7765c62bf2 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java @@ -0,0 +1,54 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity; + +import jakarta.persistence.*; + +import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.MemberBucketEntity.BUCKETISATION; + +@Entity +@Table(name = BUCKETISATION, indexes = { + @Index(columnList = "viewName"), + @Index(columnList = "fragmentId"), + @Index(columnList = "sequenceNr") +}) +public class MemberBucketEntity { + public static final String BUCKETISATION = "fragmentation_bucketisation"; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(columnDefinition = "serial") + private long id; + private String viewName; + private String fragmentId; + private String memberId; + private long sequenceNr; + + public MemberBucketEntity(String viewName, String fragmentId, String memberId, long sequenceNr) { + this.viewName = viewName; + this.fragmentId = fragmentId; + this.memberId = memberId; + this.sequenceNr = sequenceNr; + } + + protected MemberBucketEntity() { + + } + + public long getId() { + return id; + } + + public String getViewName() { + return viewName; + } + + public String getFragmentId() { + return fragmentId; + } + + public String getMemberId() { + return memberId; + } + + public long getSequenceNr() { + return sequenceNr; + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/MemberBucketEntityMapper.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/MemberBucketEntityMapper.java new file mode 100644 index 0000000000..9bef51666e --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/MemberBucketEntityMapper.java @@ -0,0 +1,20 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.MemberBucketEntity; +import org.springframework.stereotype.Component; + +@Component +public class MemberBucketEntityMapper { + + public MemberBucketEntity toMemberBucketisationEntity(BucketisedMember member) { + return new MemberBucketEntity(member.viewName().asString(), member.fragmentId(), + member.memberId(), member.sequenceNr()); + } + + public BucketisedMember toBucketisedMember(MemberBucketEntity entity) { + return new BucketisedMember(entity.getMemberId(), ViewName.fromString(entity.getViewName()), + entity.getFragmentId(), entity.getSequenceNr()); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/SequenceEntityMapper.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/SequenceEntityMapper.java index 3d1540b0fb..6353c3a43e 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/SequenceEntityMapper.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/SequenceEntityMapper.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.SequenceEntity; import org.springframework.stereotype.Component; diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java index e6e7b134fd..a478c46390 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java @@ -16,40 +16,40 @@ @Primary public interface FragmentEntityRepository extends JpaRepository { - Optional findLdesFragmentEntityByRootAndViewName(Boolean root, String viewName); + Optional findLdesFragmentEntityByRootAndViewName(Boolean root, String viewName); - List findAllByImmutableAndViewName(Boolean immutable, String viewName); + List findAllByImmutableAndViewName(Boolean immutable, String viewName); - Optional findByImmutableAndParentId(boolean immutable, String parentId); - Optional findAllByImmutableAndParentId(boolean immutable, String parentId); + Optional findByImmutableAndParentId(boolean immutable, String parentId); + Optional findAllByImmutableAndParentId(boolean immutable, String parentId); - Stream findAllByViewName(String viewName); + Stream findAllByViewName(String viewName); - List removeByViewName(String viewName); + List removeByViewName(String viewName); - Long deleteAllByCollectionName(String collectionName); + Long deleteAllByCollectionName(String collectionName); - List findAllByRelations_TreeNode(String fragmentId); + List findAllByRelations_TreeNode(String fragmentId); - @Modifying - @Transactional - @Query(value = "WITH RECURSIVE children AS (" + - "SELECT id " + - "FROM fragmentation_fragment " + - "WHERE id = :parentId " + - "UNION ALL " + - "SELECT f.id " + - "FROM fragmentation_fragment f " + - "INNER JOIN children c ON f.parent_id = c.id) " + - "UPDATE fragmentation_fragment " + - "SET immutable = true " + - "WHERE id IN (SELECT id FROM children)", nativeQuery = true) - int closeChildren(@Param("parentId") String parentId); + @Modifying + @Transactional + @Query(value = "WITH RECURSIVE children AS (" + + "SELECT id " + + "FROM fragmentation_fragment " + + "WHERE id = :parentId " + + "UNION ALL " + + "SELECT f.id " + + "FROM fragmentation_fragment f " + + "INNER JOIN children c ON f.parent_id = c.id) " + + "UPDATE fragmentation_fragment " + + "SET immutable = true " + + "WHERE id IN (SELECT id FROM children)", nativeQuery = true) + int closeChildren(@Param("parentId") String parentId); - @Modifying - @Query(value = "UPDATE fragmentation_fragment SET nr_of_members_added = nr_of_members_added + :memberCount WHERE id = :id", - nativeQuery = true) - void incrementNrOfMembersAdded(@Param("id") String id, @Param("memberCount") Integer memberCount); + @Modifying + @Query(value = "UPDATE fragmentation_fragment SET nr_of_members_added = nr_of_members_added + :memberCount WHERE id = :id", + nativeQuery = true) + void incrementNrOfMembersAdded(@Param("id") String id, @Param("memberCount") Integer memberCount); - Stream findByDeleteTimeNotNull(); + Stream findByDeleteTimeNotNull(); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/MemberBucketEntityRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/MemberBucketEntityRepository.java new file mode 100644 index 0000000000..1f7ef6f170 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/MemberBucketEntityRepository.java @@ -0,0 +1,12 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.MemberBucketEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface MemberBucketEntityRepository extends JpaRepository { + List findAllByViewNameAndSequenceNr(String viewName, Long sequenceNr); + void deleteAllByViewName (String viewName); + void deleteAllByViewNameStartingWith (String collectionName); +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/SequenceEntityRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/SequenceEntityRepository.java index c986709616..fa1dc47aa7 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/SequenceEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/SequenceEntityRepository.java @@ -7,5 +7,5 @@ @Primary public interface SequenceEntityRepository extends JpaRepository { - void deleteAllByViewNameStartingWith(String collectionName); + void deleteAllByViewNameStartingWith(String collectionName); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/PostgresFragmentationIntegrationTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/PostgresFragmentationIntegrationTest.java index fd33d91b46..f71127238e 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/PostgresFragmentationIntegrationTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/PostgresFragmentationIntegrationTest.java @@ -1,6 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.FragmentEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentSequenceRepository; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.repositories.MemberRepository; @@ -36,6 +37,9 @@ public class PostgresFragmentationIntegrationTest { @Autowired public FragmentSequenceRepository fragmentSequenceRepository; + @Autowired + public BucketisedMemberRepository bucketisedMemberRepository; + @TestConfiguration public static class EventStreamControllerTestConfiguration { @@ -53,7 +57,5 @@ public MemberRepository memberRepository() { public MemberPropertiesRepository memberPropertiesRepository() { return Mockito.mock(MemberPropertiesRepository.class); } - } - } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositoryIT.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositoryIT.java new file mode 100644 index 0000000000..c019a7c96f --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositoryIT.java @@ -0,0 +1,12 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.bucketisedmember; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.PostgresFragmentationIntegrationTest; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource("features/bucket") +public class BucketRepositoryIT extends PostgresFragmentationIntegrationTest { +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositorySteps.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositorySteps.java new file mode 100644 index 0000000000..a51679ab64 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/bucketisedmember/BucketRepositorySteps.java @@ -0,0 +1,85 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.bucketisedmember; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.PostgresFragmentationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import io.cucumber.java.Before; +import io.cucumber.java.DataTableType; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class BucketRepositorySteps extends PostgresFragmentationIntegrationTest { + + private List members; + + @DataTableType + public BucketisedMember bucketisedMemberEntryTransformer(Map row) { + return new BucketisedMember(row.get("memberId"), ViewName.fromString(row.get("viewName")), + row.get("fragmentId"), Long.parseLong(row.get("sequenceNr"))); + } + + @Before + public void setup() { + members = List.of(); + bucketisedMemberRepository.deleteByCollection("mobility-hindrances"); + bucketisedMemberRepository.deleteByCollection("parcels"); + } + + @Given("The following bucketisedMembers") + public void theFollowingMembers(List members) { + this.members = members; + } + + @When("I save the bucketisedMembers using the BucketisedMemberRepository") + public void iSaveTheMembers() { + bucketisedMemberRepository.insertAll(members); + } + + @When("I delete the members of collection {string}") + public void iDeleteBucketisedMembersOfTheCollection(String collectionName) { + bucketisedMemberRepository.deleteByCollection(collectionName); + } + + @When("I delete the members of view {string}") + public void iDeleteBucketisedMembersOfTheView(String viewName) { + bucketisedMemberRepository.deleteByViewName(ViewName.fromString(viewName)); + } + + @Then("The BucketisedMemberRepository contains all the members") + public void membersArePresent() { + members.forEach(bucketisedMember -> { + assertFalse( + bucketisedMemberRepository + .getFirstUnallocatedMember(bucketisedMember.viewName(), bucketisedMember.sequenceNr()).isEmpty()); + }); + bucketisedMemberRepository.insertAll(members); + } + + @Then("The BucketisedMemberRepository does not contain the members of collection {string}") + public void membersOfCollectionAreNotPresent(String collection) { + members.forEach(bucketisedMember -> { + assertFalse( + bucketisedMember.viewName().getCollectionName().equals(collection) ^ + bucketisedMemberRepository + .getFirstUnallocatedMember(bucketisedMember.viewName(), bucketisedMember.sequenceNr()).isEmpty()); + }); + bucketisedMemberRepository.insertAll(members); + } + + @Then("The BucketisedMemberRepository does not contain the members of view {string}") + public void membersAreNotPresent(String viewName) { + members.forEach(bucketisedMember -> { + assertFalse( + bucketisedMember.viewName().asString().equals(viewName) ^ + bucketisedMemberRepository + .getFirstUnallocatedMember(bucketisedMember.viewName(), bucketisedMember.sequenceNr()).isEmpty()); + }); + bucketisedMemberRepository.insertAll(members); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/sequence/FragmentSequenceRepositorySteps.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/sequence/FragmentSequenceRepositorySteps.java index e7b48b0302..15db4b9524 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/sequence/FragmentSequenceRepositorySteps.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/sequence/FragmentSequenceRepositorySteps.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.PostgresFragmentationIntegrationTest; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/features/bucket/bucket.feature b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/features/bucket/bucket.feature new file mode 100644 index 0000000000..3491e4a7c3 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/features/bucket/bucket.feature @@ -0,0 +1,34 @@ +Feature: BucketisedMemberRepository + As a user + I want to interact with the BucketisedMemberRepository to + create, retrieve and delete BucketisedMembers + + Scenario: Saving and retrieving BucketisedMembers + Given The following bucketisedMembers + | memberId | viewName | fragmentId | sequenceNr | + | member1 | mobility-hindrances/by-time-and-page | /mobility-hindrances/by-time-and-page?year=2023 | 1 | + | member1 | mobility-hindrances/by-time-and-page2 | /mobility-hindrances/by-time-and-page?year=2024 | 1 | + | member2 | mobility-hindrances/by-time-and-page | /mobility-hindrances/by-time-and-page?year=2023 | 2 | + | member3 | parcels/by-time-and-page | /parcels/by-time-and-page?year=2023 | 1 | + When I save the bucketisedMembers using the BucketisedMemberRepository + Then The BucketisedMemberRepository contains all the members + + Scenario: Deleting a collection + Given The following bucketisedMembers + | memberId | viewName | fragmentId | sequenceNr | + | member1 | mobility-hindrances/by-time-and-page | /mobility-hindrances/by-time-and-page?year=2023 | 1 | + | member1 | mobility-hindrances/by-time-and-page2 | /mobility-hindrances/by-time-and-page?year=2024 | 1 | + | member2 | parcels/by-time-and-page | /parcels/by-time-and-page?year=2023 | 1 | + When I save the bucketisedMembers using the BucketisedMemberRepository + And I delete the members of collection "mobility-hindrances" + Then The BucketisedMemberRepository does not contain the members of collection "mobility-hindrances" + + Scenario: Deleting a view + Given The following bucketisedMembers + | memberId | viewName | fragmentId | sequenceNr | + | member1 | mobility-hindrances/by-time-and-page | /mobility-hindrances/by-time-and-page?year=2023 | 1 | + | member1 | mobility-hindrances/by-time-and-page2 | /mobility-hindrances/by-time-and-page?year=2024 | 1 | + | member2 | parcels/by-time-and-page | /parcels/by-time-and-page?year=2023 | 1 | + When I save the bucketisedMembers using the BucketisedMemberRepository + And I delete the members of view "mobility-hindrances/by-time-and-page" + Then The BucketisedMemberRepository does not contain the members of view "mobility-hindrances/by-time-and-page" \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/pom.xml b/ldes-server-infra-postgres/postgres-liquibase/pom.xml new file mode 100644 index 0000000000..f6575c0bb5 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/pom.xml @@ -0,0 +1,21 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-infra-postgres + 3.1.0-SNAPSHOT + + + postgres-liquibase + + + + org.liquibase + liquibase-core + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/admin.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/admin.xml new file mode 100644 index 0000000000..f8fb026bad --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/admin.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fetch.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fetch.xml new file mode 100644 index 0000000000..d7337f9dd0 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fetch.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fragmentation.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fragmentation.xml new file mode 100644 index 0000000000..50cc10b96d --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/fragmentation.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/ingest.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/ingest.xml new file mode 100644 index 0000000000..c4ccc131b6 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/ingest.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/master.xml new file mode 100644 index 0000000000..0bc4e74f5a --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/master.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/retention.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/retention.xml new file mode 100644 index 0000000000..097d3df1b7 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_0/retention.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.sql new file mode 100644 index 0000000000..b303198439 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.sql @@ -0,0 +1,3 @@ +insert into fragmentation_bucketisation (view_name, fragment_id, member_id, sequence_nr) +SELECT fetch_allocation.view_name, fetch_allocation.fragment_id, fetch_allocation.member_id, ingest_ldesmember.sequence_nr FROM fetch_allocation +JOIN ingest_ldesmember on fetch_allocation.member_id=ingest_ldesmember.id \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.xml new file mode 100644 index 0000000000..bb5f93e3a0 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/bucketisation.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/master.xml new file mode 100644 index 0000000000..3fbc846a52 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/master.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.sql new file mode 100644 index 0000000000..92b78f1b0b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.sql @@ -0,0 +1,2 @@ +insert into pagination_sequence (view_name, last_processed_sequence) +SELECT view_name, last_processed_sequence FROM fragmentation_sequence \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.xml new file mode 100644 index 0000000000..af21d9d10b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_0_1/pagination_sequence.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml new file mode 100644 index 0000000000..bd6964cd6d --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml new file mode 100644 index 0000000000..55e3a9d066 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-infra-postgres + 3.1.0-SNAPSHOT + + + postgres-pagination-repository + + + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-domain + + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-pagination + ${project.version} + compile + + + be.vlaanderen.informatievlaanderen.vsds + postgres-fragmentation-repository + ${project.version} + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSequencePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSequencePostgresRepository.java new file mode 100644 index 0000000000..cff08b435a --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSequencePostgresRepository.java @@ -0,0 +1,51 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.mapper.PaginationSequenceEntityMapper; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PaginationSequenceEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Optional; + +@Component +@Primary +public class PaginationSequencePostgresRepository implements PaginationSequenceRepository { + + private static final String COLLECTION_VIEW_SEPARATOR = "/"; + + private final PaginationSequenceEntityRepository repository; + private final PaginationSequenceEntityMapper mapper = new PaginationSequenceEntityMapper(); + + public PaginationSequencePostgresRepository(PaginationSequenceEntityRepository repository) { + this.repository = repository; + } + + @Override + public Optional findLastProcessedSequence(ViewName viewName) { + return repository + .findById(viewName.asString()) + .map(mapper::toFragmentSequence); + } + + @Override + public void saveLastProcessedSequence(FragmentSequence sequence) { + repository.save(mapper.toEntity(sequence)); + } + + @Override + @Transactional + public void deleteByViewName(ViewName viewName) { + repository.deleteById(viewName.asString()); + } + + @Override + @Transactional + public void deleteByCollection(String collectionName) { + repository.deleteAllByViewNameStartingWith(collectionName + COLLECTION_VIEW_SEPARATOR); + } + +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PaginationSequenceEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PaginationSequenceEntity.java new file mode 100644 index 0000000000..5c42770a4e --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PaginationSequenceEntity.java @@ -0,0 +1,30 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +@Entity +@Table(name = "pagination_sequence") +public class PaginationSequenceEntity { + + @Id + private String viewName; + + private long lastProcessedSequence; + + protected PaginationSequenceEntity() {} + + public PaginationSequenceEntity(String viewName, long lastProcessedSequence) { + this.viewName = viewName; + this.lastProcessedSequence = lastProcessedSequence; + } + + public String getViewName() { + return viewName; + } + + public long getLastProcessedSequence() { + return lastProcessedSequence; + } +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/PaginationSequenceEntityMapper.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/PaginationSequenceEntityMapper.java new file mode 100644 index 0000000000..c8f91b3d20 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/PaginationSequenceEntityMapper.java @@ -0,0 +1,21 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.mapper; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PaginationSequenceEntity; +import org.springframework.stereotype.Component; + +@Component +public class PaginationSequenceEntityMapper { + + public FragmentSequence toFragmentSequence(PaginationSequenceEntity entity) { + final ViewName viewName = ViewName.fromString(entity.getViewName()); + return new FragmentSequence(viewName, entity.getLastProcessedSequence()); + } + + public PaginationSequenceEntity toEntity(FragmentSequence fragmentSequence) { + String viewName = fragmentSequence.viewName().asString(); + return new PaginationSequenceEntity(viewName, fragmentSequence.sequenceNr()); + } + +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PaginationSequenceEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PaginationSequenceEntityRepository.java new file mode 100644 index 0000000000..e6d0b541e5 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PaginationSequenceEntityRepository.java @@ -0,0 +1,11 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository; + + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PaginationSequenceEntity; +import org.springframework.context.annotation.Primary; +import org.springframework.data.jpa.repository.JpaRepository; + +@Primary +public interface PaginationSequenceEntityRepository extends JpaRepository { + void deleteAllByViewNameStartingWith(String collectionName); +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationSequenceRepositorySteps.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationSequenceRepositorySteps.java new file mode 100644 index 0000000000..50a112abe3 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationSequenceRepositorySteps.java @@ -0,0 +1,49 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import io.cucumber.java.en.And; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PaginationSequenceRepositorySteps extends PostgresPaginationIntegrationTest { + + private Optional retrieveFragmentSequence; + + @When("I save a new FragmentSequence for view {string} with sequenceNr {int}") + public void iSaveANewFragmentSequenceForViewWithSequenceNr(String viewName, int sequenceNr) { + FragmentSequence fragmentSequence = new FragmentSequence(ViewName.fromString(viewName), sequenceNr); + sequenceRepository.saveLastProcessedSequence(fragmentSequence); + } + + @And("I request the last processed sequence for view {string}") + public void iRequestTheLastProcessedSequenceForView(String viewName) { + retrieveFragmentSequence = sequenceRepository.findLastProcessedSequence(ViewName.fromString(viewName)); + } + + @Then("I receive a FragmentSequence with sequenceNr {int}") + public void iReceiveAFragmentSequenceWithSequenceNr(int sequenceNr) { + assertTrue(retrieveFragmentSequence.isPresent()); + assertEquals(sequenceNr, retrieveFragmentSequence.get().sequenceNr()); + } + + @When("I delete the sequence for view {string}") + public void iDeleteTheSequenceForView(String viewName) { + sequenceRepository.deleteByViewName(ViewName.fromString(viewName)); + } + + @Then("I do not find a FragmentSequence") + public void iDoNotFindAFragmentSequence() { + assertTrue(retrieveFragmentSequence.isEmpty()); + } + + @When("I delete the sequence for collection {string}") + public void iDeleteTheSequenceForCollection(String collectionName) { + sequenceRepository.deleteByCollection(collectionName); + } +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PostgresPaginationIntegrationTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PostgresPaginationIntegrationTest.java new file mode 100644 index 0000000000..7eeac2cdce --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PostgresPaginationIntegrationTest.java @@ -0,0 +1,72 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.FragmentEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.MemberBucketEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.SequenceEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.ingest.repositories.MemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; +import io.cucumber.spring.CucumberContextConfiguration; +import io.micrometer.observation.ObservationRegistry; +import io.zonky.test.db.AutoConfigureEmbeddedDatabase; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; + +@CucumberContextConfiguration +@EnableAutoConfiguration +@DataJpaTest +@AutoConfigureEmbeddedDatabase +@ContextConfiguration(classes = {ApplicationContext.class}) +@ActiveProfiles("postgres-test") +@ComponentScan(value = {"be.vlaanderen.informatievlaanderen.ldes.server"}) +@Import(PostgresPaginationIntegrationTest.PaginationTestConfiguration.class) +@SuppressWarnings("java:S2187") +public class PostgresPaginationIntegrationTest { + + @Autowired + public PaginationSequenceRepository sequenceRepository; + + @TestConfiguration + public static class PaginationTestConfiguration { + + @Bean + public ObservationRegistry observationRegistry() { + return ObservationRegistry.NOOP; + } + + @Bean + public MemberBucketEntityRepository memberBucketRepository() { + return Mockito.mock(MemberBucketEntityRepository.class); + } + + @Bean + public FragmentEntityRepository fragmentEntityRepositoryRepository() { + return Mockito.mock(FragmentEntityRepository.class); + } + + @Bean + public MemberRepository memberRepository() { + return Mockito.mock(MemberRepository.class); + } + + @Bean + public SequenceEntityRepository sequenceEntityRepository() { + return Mockito.mock(SequenceEntityRepository.class); + } + + @Bean + public MemberPropertiesRepository memberPropertiesRepository() { + return Mockito.mock(MemberPropertiesRepository.class); + } + + } +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/pagination-sequence.feature b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/pagination-sequence.feature new file mode 100644 index 0000000000..42edda7d90 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/pagination-sequence.feature @@ -0,0 +1,36 @@ +Feature: PaginationSequenceRepository + As a user + I want to interact with the PaginationSequenceRepository to + create, retrieve and delete fragmentation sequences + + Scenario: Saving and updating a FragmentSequence + When I save a new FragmentSequence for view "mobility-hindrances/by-page" with sequenceNr 200 + And I request the last processed sequence for view "mobility-hindrances/by-page" + Then I receive a FragmentSequence with sequenceNr 200 + When I save a new FragmentSequence for view "mobility-hindrances/by-page" with sequenceNr 300 + And I request the last processed sequence for view "mobility-hindrances/by-page" + Then I receive a FragmentSequence with sequenceNr 300 + + Scenario: Deleting a FragmentSequence by viewName + When I save a new FragmentSequence for view "mobility-hindrances/by-page" with sequenceNr 200 + And I request the last processed sequence for view "mobility-hindrances/by-page" + Then I receive a FragmentSequence with sequenceNr 200 + When I delete the sequence for view "mobility-hindrances/by-page" + And I request the last processed sequence for view "mobility-hindrances/by-page" + Then I do not find a FragmentSequence + + Scenario: Deleting a FragmentSequence by collectionName + When I save a new FragmentSequence for view "mobility-hindrances/by-page" with sequenceNr 200 + And I request the last processed sequence for view "mobility-hindrances/by-page" + Then I receive a FragmentSequence with sequenceNr 200 + And I save a new FragmentSequence for view "mobility-hindrances/by-location" with sequenceNr 300 + And I request the last processed sequence for view "mobility-hindrances/by-location" + Then I receive a FragmentSequence with sequenceNr 300 + And I save a new FragmentSequence for view "mobility-hindrances-alt/by-page" with sequenceNr 100 + When I delete the sequence for collection "mobility-hindrances" + And I request the last processed sequence for view "mobility-hindrances/by-page" + Then I do not find a FragmentSequence + And I request the last processed sequence for view "mobility-hindrances/by-location" + Then I do not find a FragmentSequence + And I request the last processed sequence for view "mobility-hindrances-alt/by-page" + Then I receive a FragmentSequence with sequenceNr 100 \ No newline at end of file diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index 06da56e16a..733bff4f61 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -62,7 +62,13 @@ be.vlaanderen.informatievlaanderen.vsds - ldes-fragmentisers-pagination + ldes-server-pagination + ${project.version} + test + + + be.vlaanderen.informatievlaanderen.vsds + postgres-pagination-repository ${project.version} test diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java index 7eacdee577..45bcdda6df 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java @@ -71,7 +71,7 @@ public void iFetchTheNextFragmentThroughTheFirst(String relation) throws Excepti @Then("this fragment only has {int} {string} relation") public void thisFragmentOnlyHasOne(int expectedRelationCount, String relation) { - await().atMost(Duration.of(40, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); int relationCount = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) .toList().size(); @@ -88,7 +88,7 @@ public void thisFragmentIsImmutable() { @And("this fragment contains {int} members") public void thisFragmentContainsMembers(int expectedMemberCount) { - await().atMost(Duration.of(20, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); return MemberCounter.countMembers(expectedMemberCount).matches(currentFragment); }); @@ -101,7 +101,7 @@ public void thisFragmentIsNotImmutable() { @And("this fragment has no relations") public void thisFragmentHasNoRelations() { - await().atMost(Duration.of(20, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); return !currentFragment.listObjectsOfProperty(createProperty(TREE + "relation")).hasNext(); }); diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java index 092c57535e..30b25ee55d 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java @@ -243,7 +243,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l .next() .toString(); - await().atMost(Duration.ofSeconds(40)) + await().atMost(Duration.ofSeconds(120)) .until(() -> { Model fragmentPage = RDFParser.fromString( mockMvc.perform(get(fragmentUrl.formatted(collection, view)) @@ -260,7 +260,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l @And("the LDES {string} contains {int} members") public void theLDESContainsMembers(String collection, int expectedMemberCount) { - await().atMost(Duration.ofSeconds(20)) + await().atMost(Duration.ofSeconds(80)) .until(() -> memberRepository.getMemberStreamOfCollection(collection).count() == expectedMemberCount); } @@ -311,7 +311,7 @@ public void iIngestFilesOfStateObjectsFromFolderToTheCollection(int numberOfStat @When("I fetch a fragment from url {string} in a streaming way") public void iFetchAStreamingFragment(String url) { - await().atMost(Duration.ofSeconds(40)) + await().atMost(Duration.ofSeconds(80)) .until(() -> { FluxExchangeResult response = client.get() .uri(url) diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml b/ldes-server-pagination/pom.xml similarity index 57% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml rename to ldes-server-pagination/pom.xml index 9c53ee94b2..8ca3e6789c 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml +++ b/ldes-server-pagination/pom.xml @@ -1,13 +1,15 @@ - + + 4.0.0 - ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds + ldes-server 3.1.0-SNAPSHOT - 4.0.0 - ldes-fragmentisers-pagination + ldes-server-pagination @@ -23,13 +25,32 @@ org.springframework.boot spring-boot-configuration-processor - + + + ${project.groupId} + ldes-server-domain + be.vlaanderen.informatievlaanderen.vsds ldes-fragmentisers-common ${project.version} + + + + org.junit.jupiter + junit-jupiter + + + org.mockito + mockito-core + + + org.springframework.boot + spring-boot-starter-test + + ${project.artifactId} @@ -39,5 +60,4 @@ - \ No newline at end of file diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationService.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationService.java new file mode 100644 index 0000000000..3110fd8e4d --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationService.java @@ -0,0 +1,93 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.MemberAllocatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.services.OpenPageProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.springframework.context.ApplicationEventPublisher; + +import java.util.List; +import java.util.concurrent.Future; + +public class MemberPaginationService { + private final PaginationSequenceRepository sequenceRepository; + private final BucketisedMemberRepository bucketisedMemberRepository; + private final OpenPageProvider openPageProvider; + private final FragmentRepository fragmentRepository; + private final ApplicationEventPublisher eventPublisher; + private final ViewName viewName; + private long sequenceNr; + private Future task; + + public MemberPaginationService(PaginationSequenceRepository sequenceRepository, BucketisedMemberRepository bucketisedMemberRepository, + OpenPageProvider openPageProvider, FragmentRepository fragmentRepository, ApplicationEventPublisher eventPublisher, ViewName viewName) { + this.sequenceRepository = sequenceRepository; + this.bucketisedMemberRepository = bucketisedMemberRepository; + this.openPageProvider = openPageProvider; + this.fragmentRepository = fragmentRepository; + this.eventPublisher = eventPublisher; + this.viewName = viewName; + sequenceNr = determineLastProcessedSequence(viewName).sequenceNr(); + } + + public void paginateMember() { + List bucketisedMembers = getNextMember(viewName); + while(!bucketisedMembers.isEmpty()) { + List> pages = getPages(bucketisedMembers); + allocateMembers(pages); + + sequenceRepository.saveLastProcessedSequence(new FragmentSequence(viewName, ++sequenceNr)); + bucketisedMembers = getNextMember(viewName); + } + } + + public boolean isRunning() { + return task != null && !(task.isDone() || task.isCancelled()); + } + + public void stopTask() { + task.cancel(true); + } + + public void setTask(Future task) { + this.task = task; + } + + private FragmentSequence determineLastProcessedSequence(ViewName viewName) { + return sequenceRepository.findLastProcessedSequence(viewName) + .orElse(new FragmentSequence(viewName, 0)); + } + + private List getNextMember(ViewName viewName) { + return bucketisedMemberRepository.getFirstUnallocatedMember(viewName, sequenceNr + 1); + } + + private List> getPages(List bucketisedMembers) { + return bucketisedMembers.stream().map(member -> { + Fragment page = openPageProvider + .retrieveOpenFragmentOrCreateNewFragment(LdesFragmentIdentifier.fromFragmentId(member.fragmentId())); + return new ImmutablePair<>(page, member); + }).toList(); + } + + private void allocateMembers(List> pages) { + pages.forEach(pair -> { + BucketisedMember member = pair.getRight(); + Fragment page = pair.getLeft(); + eventPublisher.publishEvent(new MemberAllocatedEvent(member.memberId(), page.getViewName().getCollectionName(), + page.getViewName().getViewName(), page.getFragmentId().asDecodedFragmentId())); + }); + + pages.forEach(pair -> { + Fragment page = pair.getLeft(); + fragmentRepository.incrementNrOfMembersAdded(page.getFragmentId()); + }); + } +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreator.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreator.java new file mode 100644 index 0000000000..71a06cf161 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreator.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.config.PaginationProperties; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.services.OpenPageProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.config.PaginationConfig; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.services.PageCreator; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; + +@Component +public class MemberPaginationServiceCreator { + private final PaginationSequenceRepository sequenceRepository; + private final BucketisedMemberRepository bucketisedMemberRepository; + private final FragmentRepository fragmentRepository; + private final ApplicationEventPublisher eventPublisher; + + public MemberPaginationServiceCreator(PaginationSequenceRepository sequenceRepository, + BucketisedMemberRepository bucketisedMemberRepository, + FragmentRepository fragmentRepository, + ApplicationEventPublisher eventPublisher) { + this.sequenceRepository = sequenceRepository; + this.bucketisedMemberRepository = bucketisedMemberRepository; + this.fragmentRepository = fragmentRepository; + this.eventPublisher = eventPublisher; + } + + public MemberPaginationService createPaginationService(ViewName viewName, ViewSpecification view) { + OpenPageProvider openPageProvider = getOpenPageProvider(view.getPaginationProperties()); + + return new MemberPaginationService(sequenceRepository, bucketisedMemberRepository, + openPageProvider, fragmentRepository, eventPublisher, viewName); + } + + private OpenPageProvider getOpenPageProvider(ConfigProperties properties) { + PaginationConfig paginationConfig = createPaginationConfig(properties); + PageCreator pageFragmentCreator = getPageCreator(paginationConfig.bidirectionalRelations()); + return new OpenPageProvider(pageFragmentCreator, fragmentRepository, + paginationConfig.memberLimit()); + } + + private PageCreator getPageCreator(boolean bidirectionalRelations) { + return new PageCreator( + fragmentRepository, bidirectionalRelations); + } + + private PaginationConfig createPaginationConfig(ConfigProperties properties) { + return new PaginationConfig(Long.valueOf(properties.get(PaginationProperties.MEMBER_LIMIT)), + Boolean.parseBoolean(properties.getOrDefault(PaginationProperties.BIDIRECTIONAL_RELATIONS, "true"))); + } +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationService.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationService.java new file mode 100644 index 0000000000..082eb9714d --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationService.java @@ -0,0 +1,78 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.MemberBucketisedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; + +@Component +public class PaginationService { + private static final Logger LOGGER = LoggerFactory.getLogger(PaginationService.class); + private final MemberPaginationServiceCreator memberPaginationServiceCreator; + private final PaginationSequenceRepository sequenceRepository; + private final Map map = new HashMap<>(); + private final ExecutorService executorService; + + public PaginationService(MemberPaginationServiceCreator memberPaginationServiceCreator, PaginationSequenceRepository sequenceRepository) { + this.memberPaginationServiceCreator = memberPaginationServiceCreator; + this.sequenceRepository = sequenceRepository; + executorService = createExecutorService(); + } + + @EventListener + public void handleViewInitializationEvent(ViewInitializationEvent event) { + addToMap(event.getViewName(), event.getViewSpecification()); + } + + @EventListener + public void handleViewAddedEvent(ViewAddedEvent event) { + addToMap(event.getViewName(), event.getViewSpecification()); + } + + @EventListener + public void handleViewDeletedEvent(ViewDeletedEvent event) { + ViewName viewName = event.getViewName(); + MemberPaginationService paginationService = map.get(viewName); + if (paginationService.isRunning()) { + paginationService.stopTask(); + } + sequenceRepository.deleteByViewName(viewName); + map.remove(viewName); + } + + @EventListener + @Async + @SuppressWarnings("java:S2629") + public void handleMemberBucketisedEvent(MemberBucketisedEvent event) { + ViewName viewName = event.viewName(); + MemberPaginationService paginationService = map.get(viewName); + if (paginationService == null) { + LOGGER.warn("Missing view: {}", viewName.asString()); + } else if (!paginationService.isRunning()) { + Future task = executorService.submit(paginationService::paginateMember); + paginationService.setTask(task); + } + } + + private ExecutorService createExecutorService() { + return new ThreadPoolExecutor(0, 5, + 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(5, true), new ThreadPoolExecutor.DiscardPolicy()); + } + + private void addToMap(ViewName viewName, ViewSpecification view) { + map.put(viewName, memberPaginationServiceCreator.createPaginationService(viewName, view)); + } +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationConfig.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationConfig.java new file mode 100644 index 0000000000..fad19e0798 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationConfig.java @@ -0,0 +1,4 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.config; + +public record PaginationConfig(Long memberLimit, boolean bidirectionalRelations) { +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationProperties.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationProperties.java similarity index 71% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationProperties.java rename to ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationProperties.java index 18c4cf4d27..e634238bda 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/config/PaginationProperties.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/config/PaginationProperties.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.config; +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.config; public class PaginationProperties { private PaginationProperties() { diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/constants/PaginationConstants.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/constants/PaginationConstants.java similarity index 67% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/constants/PaginationConstants.java rename to ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/constants/PaginationConstants.java index b74bae4c18..b0a6f5c398 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/constants/PaginationConstants.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/constants/PaginationConstants.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants; +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.constants; public class PaginationConstants { diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java new file mode 100644 index 0000000000..ce1754fe02 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java @@ -0,0 +1,16 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; + +import java.util.Optional; + +public interface PaginationSequenceRepository { + Optional findLastProcessedSequence(ViewName viewName); + + void saveLastProcessedSequence(FragmentSequence sequence); + + void deleteByViewName(ViewName viewName); + + void deleteByCollection(String collectionName); +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/OpenPageProvider.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/OpenPageProvider.java similarity index 52% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/OpenPageProvider.java rename to ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/OpenPageProvider.java index 3fb1b9058e..e9787c9441 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/OpenPageProvider.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/OpenPageProvider.java @@ -1,8 +1,11 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services; +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; -import org.apache.commons.lang3.tuple.ImmutablePair; + +import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.GENERIC_TREE_RELATION; public class OpenPageProvider { private final PageCreator pageCreator; @@ -16,23 +19,32 @@ public OpenPageProvider(PageCreator pageCreator, this.memberLimit = memberLimit; } - public ImmutablePair retrieveOpenFragmentOrCreateNewFragment(Fragment parentFragment) { + public Fragment retrieveOpenFragmentOrCreateNewFragment(LdesFragmentIdentifier parentId) { return fragmentRepository - .retrieveOpenChildFragment(parentFragment.getFragmentId()) + .retrieveOpenChildFragment(parentId) .map(fragment -> { if (needsToCreateNewFragment(fragment)) { + Fragment parentFragment = fragmentRepository.retrieveFragment(parentId).orElseThrow(); Fragment newFragment = pageCreator.createNewFragment(fragment, parentFragment); fragmentRepository.saveFragment(newFragment); - return new ImmutablePair<>(newFragment, false); + return newFragment; } else { - return new ImmutablePair<>(fragment, false); + return fragment; } }) .orElseGet(() -> { + Fragment parentFragment = fragmentRepository.retrieveFragment(parentId).orElseThrow(); Fragment newFragment = pageCreator.createFirstFragment(parentFragment); fragmentRepository.saveFragment(newFragment); - return new ImmutablePair<>(newFragment, true); + + TreeRelation treeRelation = new TreeRelation("", newFragment.getFragmentId(), "", "", GENERIC_TREE_RELATION); + if (!parentFragment.containsRelation(treeRelation)) { + parentFragment.addRelation(treeRelation); + fragmentRepository.saveFragment(parentFragment); + } + + return newFragment; }); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/PageCreator.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageCreator.java similarity index 81% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/PageCreator.java rename to ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageCreator.java index ce3ef80bae..d218ae9d31 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/PageCreator.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageCreator.java @@ -1,9 +1,10 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services; +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.constants.PaginationConstants; import io.micrometer.core.instrument.Metrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,23 +13,20 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationService.LDES_SERVER_CREATE_FRAGMENTS_COUNT; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.metrics.MetricsConstants.FRAGMENTATION_STRATEGY; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.metrics.MetricsConstants.VIEW; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.PaginationStrategy.PAGINATION_FRAGMENTATION; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants.PaginationConstants.FIRST_PAGE_NUMBER; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants.PaginationConstants.PAGE_NUMBER; public class PageCreator { + public static final String PAGINATION_FRAGMENTATION = "PaginationFragmentation"; private final FragmentRepository fragmentRepository; private final boolean bidirectionalRelations; private static final Logger LOGGER = LoggerFactory.getLogger(PageCreator.class); public PageCreator(FragmentRepository fragmentRepository, boolean bidirectionalRelations) { - this.fragmentRepository = fragmentRepository; this.bidirectionalRelations = bidirectionalRelations; } public Fragment createFirstFragment(Fragment parentFragment) { - return createFragment(parentFragment, FIRST_PAGE_NUMBER); + return createFragment(parentFragment, PaginationConstants.FIRST_PAGE_NUMBER); } public Fragment createNewFragment(Fragment previousFragment, Fragment parentFragment) { @@ -40,7 +38,7 @@ public Fragment createNewFragment(Fragment previousFragment, Fragment parentFrag } private Fragment createFragment(Fragment parentFragment, String pageNumber) { - Fragment newFragment = parentFragment.createChild(new FragmentPair(PAGE_NUMBER, pageNumber)); + Fragment newFragment = parentFragment.createChild(new FragmentPair(PaginationConstants.PAGE_NUMBER, pageNumber)); String viewName = parentFragment.getViewName().asString(); Metrics.counter(LDES_SERVER_CREATE_FRAGMENTS_COUNT, VIEW, viewName, FRAGMENTATION_STRATEGY, PAGINATION_FRAGMENTATION).increment(); LOGGER.debug("Page created with id: {}", newFragment.getFragmentId()); @@ -48,7 +46,7 @@ private Fragment createFragment(Fragment parentFragment, String pageNumber) { } private String getPageNumberAndGiveIncremented(Fragment previousFragment) { - String previousPageNumber = previousFragment.getValueOfKey(PAGE_NUMBER).orElseThrow(); + String previousPageNumber = previousFragment.getValueOfKey(PaginationConstants.PAGE_NUMBER).orElseThrow(); int incremented = Integer.parseInt(previousPageNumber) + 1; return String.valueOf(incremented); } diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreatorTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreatorTest.java new file mode 100644 index 0000000000..b1f049a69c --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceCreatorTest.java @@ -0,0 +1,48 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; + +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +class MemberPaginationServiceCreatorTest { + private final ViewName VIEW_NAME = new ViewName("collection", "view"); + private final ViewSpecification VIEW = new ViewSpecification(VIEW_NAME, List.of(), List.of(), 50); + private PaginationSequenceRepository sequenceRepository; + private BucketisedMemberRepository bucketisedMemberRepository; + private FragmentRepository fragmentRepository; + private ApplicationEventPublisher eventPublisher; + private MemberPaginationServiceCreator memberPaginationServiceCreator; + + @BeforeEach + void setUp() { + sequenceRepository = mock(PaginationSequenceRepository.class); + bucketisedMemberRepository = mock(BucketisedMemberRepository.class); + fragmentRepository = mock(FragmentRepository.class); + eventPublisher = mock(ApplicationEventPublisher.class); + memberPaginationServiceCreator = new MemberPaginationServiceCreator(sequenceRepository, bucketisedMemberRepository, + fragmentRepository, eventPublisher); + } + + @Test + void when_CreatePaginationService_Then_CreateService() { + MemberPaginationService paginationService = memberPaginationServiceCreator.createPaginationService(VIEW_NAME, VIEW); + + assertThat(paginationService).hasFieldOrPropertyWithValue("viewName", VIEW_NAME) + .hasFieldOrPropertyWithValue("sequenceRepository", sequenceRepository) + .hasFieldOrPropertyWithValue("bucketisedMemberRepository", bucketisedMemberRepository) + .hasFieldOrPropertyWithValue("fragmentRepository", fragmentRepository) + .hasFieldOrPropertyWithValue("eventPublisher", eventPublisher); + } + +} \ No newline at end of file diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceTest.java new file mode 100644 index 0000000000..797eff044d --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/MemberPaginationServiceTest.java @@ -0,0 +1,65 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.MemberAllocatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketisedMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.constants.PaginationConstants; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.services.OpenPageProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.springframework.context.ApplicationEventPublisher; + +import java.util.List; + +import static org.mockito.Mockito.*; + +class MemberPaginationServiceTest { + private final ViewName VIEW_NAME = new ViewName("collection", "view"); + private final Long SEQ_NR = -1L; + private final LdesFragmentIdentifier FRAGMENT_ID = new LdesFragmentIdentifier(VIEW_NAME, List.of()); + private final BucketisedMember MEMBER = new BucketisedMember("id", VIEW_NAME, + FRAGMENT_ID.asDecodedFragmentId(), SEQ_NR); + private final Fragment FRAGMENT = new Fragment(FRAGMENT_ID); + private PaginationSequenceRepository sequenceRepository; + private BucketisedMemberRepository bucketisedMemberRepository; + private OpenPageProvider openPageProvider; + private FragmentRepository fragmentRepository; + private ApplicationEventPublisher eventPublisher; + private MemberPaginationService paginationService; + + @BeforeEach + void setUp() { + sequenceRepository = mock(PaginationSequenceRepository.class); + bucketisedMemberRepository = mock(BucketisedMemberRepository.class); + openPageProvider = Mockito.mock(OpenPageProvider.class); + fragmentRepository = mock(FragmentRepository.class); + eventPublisher = mock(ApplicationEventPublisher.class); + paginationService = new MemberPaginationService(sequenceRepository, bucketisedMemberRepository, + openPageProvider, fragmentRepository, eventPublisher, VIEW_NAME); + } + + @Test + void when_MemberPresent_Then_MemberPaginated() { + when(bucketisedMemberRepository.getFirstUnallocatedMember(VIEW_NAME, 1L)) + .thenReturn(List.of(MEMBER)); + Fragment child = FRAGMENT.createChild(new FragmentPair(PaginationConstants.PAGE_NUMBER, "1")); + when(openPageProvider.retrieveOpenFragmentOrCreateNewFragment(FRAGMENT_ID)) + .thenReturn(child); + + paginationService.paginateMember(); + + InOrder inOrder = inOrder(eventPublisher, fragmentRepository); + inOrder.verify(eventPublisher, times(1)).publishEvent(new MemberAllocatedEvent(MEMBER.memberId(), VIEW_NAME.getCollectionName(), + MEMBER.viewName().getViewName(), child.getFragmentIdString())); + inOrder.verify(fragmentRepository, times(1)).incrementNrOfMembersAdded(child.getFragmentId()); + inOrder.verifyNoMoreInteractions(); + } +} \ No newline at end of file diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationServiceTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationServiceTest.java new file mode 100644 index 0000000000..84e1f03798 --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/PaginationServiceTest.java @@ -0,0 +1,80 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.MemberBucketisedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PaginationSequenceRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import static org.mockito.Mockito.*; + +class PaginationServiceTest { + private final ViewName VIEW_NAME_1 = new ViewName("collection", "view1"); + private final ViewName VIEW_NAME_2 = new ViewName("collection", "view2"); + private MemberPaginationService service1; + private MemberPaginationService service2; + private PaginationSequenceRepository sequenceRepository; + private MemberPaginationServiceCreator memberPaginationServiceCreator; + private ExecutorService executorService; + private PaginationService paginationService; + + @BeforeEach + void setUp() { + memberPaginationServiceCreator = Mockito.mock(MemberPaginationServiceCreator.class); + executorService = mock(ExecutorService.class); + service1 = Mockito.mock(MemberPaginationService.class); + service2 = Mockito.mock(MemberPaginationService.class); + sequenceRepository = Mockito.mock(PaginationSequenceRepository.class); + paginationService = new PaginationService(memberPaginationServiceCreator, sequenceRepository); + } + + @Test + void when_MemberBucketised_Then_CorrectServiceCalled() { + when(memberPaginationServiceCreator.createPaginationService(eq(VIEW_NAME_1), any())) + .thenReturn(service1); + when(memberPaginationServiceCreator.createPaginationService(eq(VIEW_NAME_2), any())) + .thenReturn(service2); + paginationService.handleViewAddedEvent(new ViewAddedEvent(new ViewSpecification(VIEW_NAME_1, + List.of(), List.of(), 10))); + paginationService.handleViewAddedEvent(new ViewAddedEvent(new ViewSpecification(VIEW_NAME_2, + List.of(), List.of(), 10))); + when(service1.isRunning()).thenReturn(false); + when(service2.isRunning()).thenReturn(false); + when(executorService.submit(any(Callable.class))).thenReturn(new CompletableFuture<>()); + + paginationService.handleMemberBucketisedEvent(new MemberBucketisedEvent(VIEW_NAME_1)); + + InOrder inOrder = inOrder(executorService, service1, service2); + inOrder.verify(service1).isRunning(); + inOrder.verify(service1).setTask(any()); + } + + @Test + void when_ViewDeleted_Then_ServiceRemoved() { + when(memberPaginationServiceCreator.createPaginationService(eq(VIEW_NAME_1), any())) + .thenReturn(service1); + when(memberPaginationServiceCreator.createPaginationService(eq(VIEW_NAME_2), any())) + .thenReturn(service2); + paginationService.handleViewInitializationEvent(new ViewInitializationEvent(new ViewSpecification(VIEW_NAME_1, + List.of(), List.of(), 10))); + when(service1.isRunning()).thenReturn(true); + + paginationService.handleViewDeletedEvent(new ViewDeletedEvent(VIEW_NAME_1)); + + InOrder inOrder = inOrder(service1, service2, sequenceRepository); + inOrder.verify(service1).isRunning(); + inOrder.verify(service1).stopTask(); + inOrder.verify(sequenceRepository).deleteByViewName(VIEW_NAME_1); + } +} \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/OpenPageProviderTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/OpenPageProviderTest.java similarity index 74% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/OpenPageProviderTest.java rename to ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/OpenPageProviderTest.java index 81d441ea6e..35e9bd05e9 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/OpenPageProviderTest.java +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/OpenPageProviderTest.java @@ -1,26 +1,26 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services; +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; -import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.mockito.InOrder; +import org.mockito.Mockito; import java.util.List; import java.util.Optional; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants.PaginationConstants.PAGE_NUMBER; +import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.constants.PaginationConstants.PAGE_NUMBER; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; class OpenPageProviderTest { - private final PageCreator pageCreator = mock(PageCreator.class); + private final PageCreator pageCreator = Mockito.mock(PageCreator.class); private final FragmentRepository fragmentRepository = mock(FragmentRepository.class); private OpenPageProvider openPageProvider; private static Fragment PARENT_FRAGMENT; @@ -38,21 +38,25 @@ void setUp() { void when_NoFragmentExists_thenFirstFragmentIsCreated() { Fragment createdFragment = PARENT_FRAGMENT.createChild(new FragmentPair(PAGE_NUMBER, "1")); + when(fragmentRepository.retrieveFragment(PARENT_FRAGMENT.getFragmentId())) + .thenReturn(Optional.of(PARENT_FRAGMENT)); when(fragmentRepository.retrieveOpenChildFragment(PARENT_FRAGMENT.getFragmentId())) .thenReturn(Optional.empty()); when(pageCreator.createFirstFragment(PARENT_FRAGMENT)) .thenReturn(createdFragment); - Pair ldesFragment = openPageProvider - .retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT); + Fragment ldesFragment = openPageProvider + .retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT.getFragmentId()); - assertTrue(ldesFragment.getRight()); - assertEquals(createdFragment, ldesFragment.getKey()); + assertEquals(createdFragment, ldesFragment); InOrder inOrder = inOrder(fragmentRepository, pageCreator); inOrder.verify(fragmentRepository, times(1)).retrieveOpenChildFragment(PARENT_FRAGMENT.getFragmentId()); + inOrder.verify(fragmentRepository, + times(1)).retrieveFragment(PARENT_FRAGMENT.getFragmentId()); inOrder.verify(pageCreator, times(1)).createFirstFragment(PARENT_FRAGMENT); inOrder.verify(fragmentRepository, times(1)).saveFragment(createdFragment); + inOrder.verify(fragmentRepository, times(1)).saveFragment(PARENT_FRAGMENT); inOrder.verifyNoMoreInteractions(); } @@ -64,11 +68,10 @@ void when_AnIncompleteOpenFragmentExists_thenFragmentIsReturned() { when(fragmentRepository.retrieveOpenChildFragment(PARENT_FRAGMENT.getFragmentId())) .thenReturn(Optional.of(existingFragment)); - Pair ldesFragment = openPageProvider - .retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT); + Fragment ldesFragment = openPageProvider + .retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT.getFragmentId()); - assertFalse(ldesFragment.getRight()); - assertEquals(existingFragment, ldesFragment.getKey()); + assertEquals(existingFragment, ldesFragment); InOrder inOrder = inOrder(fragmentRepository, pageCreator); inOrder.verify(fragmentRepository, times(1)).retrieveOpenChildFragment(PARENT_FRAGMENT.getFragmentId()); @@ -84,15 +87,16 @@ void when_AFullFragmentExists_thenANewFragmentIsCreatedAndReturned() { false, 3, List.of(), null); Fragment newFragment = PARENT_FRAGMENT.createChild(new FragmentPair(PAGE_NUMBER, "3")); + when(fragmentRepository.retrieveFragment(PARENT_FRAGMENT.getFragmentId())) + .thenReturn(Optional.of(PARENT_FRAGMENT)); when(fragmentRepository.retrieveOpenChildFragment(PARENT_FRAGMENT.getFragmentId())) .thenReturn(Optional.of(completeFragment)); when(pageCreator.createNewFragment(completeFragment, PARENT_FRAGMENT)).thenReturn(newFragment); - Pair ldesFragment = openPageProvider - .retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT); + Fragment ldesFragment = openPageProvider + .retrieveOpenFragmentOrCreateNewFragment(PARENT_FRAGMENT.getFragmentId()); - assertFalse(ldesFragment.getRight()); - assertEquals(newFragment, ldesFragment.getKey()); + assertEquals(newFragment, ldesFragment); InOrder inOrder = inOrder(fragmentRepository, pageCreator); inOrder.verify(fragmentRepository, times(1)).retrieveOpenChildFragment(PARENT_FRAGMENT.getFragmentId()); diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/PageCreatorTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageCreatorTest.java similarity index 88% rename from ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/PageCreatorTest.java rename to ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageCreatorTest.java index 0a8fe84318..32575631d2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/pagination/services/PageCreatorTest.java +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageCreatorTest.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.services; +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; @@ -12,8 +12,8 @@ import java.util.List; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants.PaginationConstants.FIRST_PAGE_NUMBER; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.pagination.constants.PaginationConstants.PAGE_NUMBER; +import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.constants.PaginationConstants.FIRST_PAGE_NUMBER; +import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.constants.PaginationConstants.PAGE_NUMBER; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.*; diff --git a/pom.xml b/pom.xml index d218e839f3..c125a057e0 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,7 @@ ldes-server-compaction ldes-server-integration-test ldes-server-instrumentation + ldes-server-pagination From 5586c0eda4a6de65546506cd8774454f6ffd3ef9 Mon Sep 17 00:00:00 2001 From: pj-cegeka <119848850+pj-cegeka@users.noreply.github.com> Date: Fri, 31 May 2024 10:18:18 +0200 Subject: [PATCH 03/15] fix: add liquibase profile (#1297) --- ldes-server-application/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index ac19a0afc2..b93a815e62 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -225,5 +225,15 @@ + + liquibase-changesets + + + be.vlaanderen.informatievlaanderen.vsds + postgres-liquibase + ${project.version} + + + From 0c37c5e1ed78bb8119bcbd8d0e72e00c68e4d624 Mon Sep 17 00:00:00 2001 From: ferre-vaes <61710181+ferre-vaes@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:45:46 +0200 Subject: [PATCH 04/15] feat: VSDSPUB-1283: Close LDES (#1296) * feat: Extend FragmentRepository with update query to make fragments immutable by collection * feat: New admin endpoint to close an event stream * feat: Integration test for closure of LDES * feat: validate for closed collection on member ingestion * feat: liquibase to add column to eventstream table * feat: fix member ingestion validation * feat: fix member ingestion validation --- .../fragmentation/FragmentationService.java | 14 +- .../repository/FragmentRepository.java | 2 + .../FragmentationServiceTest.java | 13 + .../services/EventStreamService.java | 2 + .../services/EventStreamServiceImpl.java | 15 +- .../AdminEventStreamsRestController.java | 9 +- .../OpenApiAdminEventStreamsController.java | 5 + .../services/EventStreamServiceImplTest.java | 525 +++++++++--------- .../AdminEventStreamsRestControllerTest.java | 22 + .../events/admin/EventStreamClosedEvent.java | 4 + .../ldes/server/domain/model/EventStream.java | 19 +- .../eventstream/entity/EventStreamEntity.java | 15 +- .../service/EventStreamConverter.java | 4 +- .../postgres/FragmentPostgresRepository.java | 9 +- .../repository/FragmentEntityRepository.java | 7 +- .../FragmentPostgresRepositoryTest.java | 13 +- .../3_1_0/eventstream-closed-column.xml | 13 + .../resources/db/changelog/3_1_0/master.xml | 6 + .../main/resources/db/changelog/master.xml | 1 + .../ldes/server/FragmentationSteps.java | 226 ++++---- .../ldes/server/LdesServerSteps.java | 6 + .../fragmentation/fragmentation.feature | 18 +- .../ldes/server/rest/config/RestConfig.java | 2 +- .../MemberIngestValidator.java | 24 +- .../MemberIngestValidatorTest.java | 13 + 25 files changed, 613 insertions(+), 374 deletions(-) create mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/admin/EventStreamClosedEvent.java create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/eventstream-closed-column.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/master.xml diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java index 29e6df8279..660e93c209 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java @@ -1,6 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamClosedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest.MembersIngestedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; @@ -10,10 +12,13 @@ public class FragmentationService { public static final String LDES_SERVER_CREATE_FRAGMENTS_COUNT = "ldes_server_create_fragments_count"; private final FragmentationStrategyCollection fragmentationStrategyCollection; + private final FragmentRepository fragmentRepository; - public FragmentationService(FragmentationStrategyCollection fragmentationStrategyCollection) { + public FragmentationService(FragmentationStrategyCollection fragmentationStrategyCollection, + FragmentRepository fragmentRepository) { this.fragmentationStrategyCollection = fragmentationStrategyCollection; - } + this.fragmentRepository = fragmentRepository; + } @EventListener public void executeFragmentation(MembersIngestedEvent event) { @@ -22,4 +27,9 @@ public void executeFragmentation(MembersIngestedEvent event) { .forEach(FragmentationStrategyExecutor::execute); } + @EventListener + public void markFragmentsImmutableInCollection(EventStreamClosedEvent event) { + fragmentRepository.markFragmentsImmutableInCollection(event.collectionName()); + } + } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentRepository.java index 4dbb07b26f..fc982af806 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentRepository.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/FragmentRepository.java @@ -67,4 +67,6 @@ Optional retrieveMutableFragment(String viewName, * @param fragment The fragment whose children should become immutable */ void makeChildrenImmutable(Fragment fragment); + + void markFragmentsImmutableInCollection(String collectionName); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java index a243fc8f35..5eb758498a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java @@ -1,6 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamClosedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest.MembersIngestedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.FragmentRepository; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -18,6 +20,9 @@ class FragmentationServiceTest { @Mock private FragmentationStrategyCollection fragmentationStrategyCollection; + @Mock + private FragmentRepository fragmentRepository; + @InjectMocks private FragmentationService fragmentationService; @@ -36,4 +41,12 @@ void when_MemberIngestedEvent_then_AllFragmentationExecutorsFromThisCollection_s verify(executorB).execute(); } + @Test + void when_EventStreamClosedEvent_then_FragmentsAreMadeImmutable() { + EventStreamClosedEvent event = new EventStreamClosedEvent("collectionName"); + + fragmentationService.markFragmentsImmutableInCollection(event); + + verify(fragmentRepository).markFragmentsImmutableInCollection("collectionName"); + } } diff --git a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamService.java b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamService.java index 1500277265..d0d081a773 100644 --- a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamService.java +++ b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamService.java @@ -13,4 +13,6 @@ public interface EventStreamService extends EventStreamServiceSpi { EventStreamTO createEventStream(EventStreamTO eventStream); void updateEventSource(String collectionName, List eventSourceModel); + + void closeEventStream(String collectionName); } diff --git a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImpl.java b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImpl.java index 27b14a1e92..bae16fdb1c 100644 --- a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImpl.java +++ b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImpl.java @@ -9,6 +9,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.shacl.services.ShaclShapeService; import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.view.service.ViewService; import be.vlaanderen.informatievlaanderen.ldes.server.admin.spi.EventStreamTO; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamClosedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; @@ -53,8 +54,7 @@ public List retrieveAllEventStreams() { @Override public EventStreamTO retrieveEventStream(String collectionName) { - EventStream eventStream = eventStreamRepository.retrieveEventStream(collectionName) - .orElseThrow(() -> new MissingResourceException("eventstream", collectionName)); + EventStream eventStream = getEventStream(collectionName); return mapToEventStreamTO(eventStream); } @@ -93,6 +93,12 @@ public void updateEventSource(String collectionName, List eventSourceMode eventSourceService.saveEventSource(collectionName, eventSourceModel); } + @Override + public void closeEventStream(String collectionName) { + EventStream eventStream = getEventStream(collectionName); + eventPublisher.publishEvent(new EventStreamClosedEvent(eventStream.getCollection())); + } + private EventStreamTO mapToEventStreamTO(EventStream eventStream) { List views = viewService.getViewsByCollectionName(eventStream.getCollection()); ShaclShape shaclShape = shaclShapeService.retrieveShaclShape(eventStream.getCollection()); @@ -105,6 +111,11 @@ private EventStreamTO mapToEventStreamTO(EventStream eventStream) { dataset.orElse(null)); } + private EventStream getEventStream(String collectionName) { + return eventStreamRepository.retrieveEventStream(collectionName) + .orElseThrow(() -> new MissingResourceException("eventstream", collectionName)); + } + private void delete(String collectionName) { eventStreamRepository.deleteEventStream(collectionName); eventPublisher.publishEvent(new EventStreamDeletedEvent(collectionName)); diff --git a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestController.java b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestController.java index 07e6b5df4f..29ed9f3c99 100644 --- a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestController.java +++ b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestController.java @@ -79,6 +79,14 @@ public void deleteEventStream(@PathVariable String collectionName) { log.atInfo().log("FINISHED deleting collection {}", collectionName); } + @Override + @PutMapping("/{collectionName}") + public void closeEventStream(@PathVariable String collectionName) { + log.atInfo().log("START closing collection {}", collectionName); + eventStreamService.closeEventStream(collectionName); + log.atInfo().log("FINISHED closing collection {}", collectionName); + } + @Override @PutMapping("/{collectionName}/eventsource") public void updateEventSource(@PathVariable String collectionName, @RequestBody Model eventSourceModel) { @@ -86,5 +94,4 @@ public void updateEventSource(@PathVariable String collectionName, @RequestBody List retentionPolicies = retentionModelExtractor.extractRetentionStatements(eventSourceModel); eventStreamService.updateEventSource(collectionName, retentionPolicies); } - } diff --git a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/OpenApiAdminEventStreamsController.java b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/OpenApiAdminEventStreamsController.java index fb96be08f9..21717d7387 100644 --- a/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/OpenApiAdminEventStreamsController.java +++ b/ldes-server-admin/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/OpenApiAdminEventStreamsController.java @@ -187,6 +187,11 @@ EventStreamTO createEventStream( @ApiResponse(responseCode = "404", description = "Event Stream with provided collection name could not be found") void deleteEventStream(@Parameter(example = "event-stream") String collectionName); + @Operation(summary = "Close an Event Stream") + @ApiResponse(responseCode = "200", description = "Event Stream is successfully closed and all related fragments are made immutable") + @ApiResponse(responseCode = "404", description = "Event Stream with provided collection name could not be found") + void closeEventStream(@Parameter(example = "event-stream") String collectionName); + @Operation(summary = "Update the Event Source of an Event Stream") @ApiResponse(responseCode = "200", description = "Event Source has been successfully updated", content = { @Content(mediaType = contentTypeTurtle, schema = @Schema(implementation = String.class), examples = @ExampleObject(value = """ diff --git a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImplTest.java b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImplTest.java index 2d76d0df00..507cde5b28 100644 --- a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImplTest.java +++ b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/domain/eventstream/services/EventStreamServiceImplTest.java @@ -10,6 +10,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.view.exception.DuplicateRetentionException; import be.vlaanderen.informatievlaanderen.ldes.server.admin.domain.view.service.ViewService; import be.vlaanderen.informatievlaanderen.ldes.server.admin.spi.EventStreamTO; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamClosedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.EventStream; @@ -40,259 +41,281 @@ @ExtendWith(MockitoExtension.class) class EventStreamServiceImplTest { - private static final String COLLECTION = "collection"; - private static final String TIMESTAMP_PATH = "generatedAt"; - private static final String VERSION_OF_PATH = "isVersionOf"; - private static final boolean VERSION_CREATION_ENABLED = false; - private static final EventStream EVENT_STREAM = new EventStream(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, VERSION_CREATION_ENABLED); - private static final EventStreamTO EVENT_STREAM_RESPONSE = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, - VERSION_OF_PATH, VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of()); - private DcatDataset dataset; - private EventStreamTO eventStreamTOWithDataset; - - @Mock - private EventStreamRepository eventStreamRepository; - @Mock - private ApplicationEventPublisher eventPublisher; - @Captor - ArgumentCaptor deletedEventArgumentCaptor; - @Mock - private ViewService viewService; - @Mock - private ShaclShapeService shaclShapeService; - @Mock - private DcatDatasetService dcatDatasetService; - @Mock - private DcatServerService dcatServerService; - @Mock - private EventSourceService eventSourceService; - - private EventStreamService service; - - @BeforeEach - void setUp() throws URISyntaxException { - service = new EventStreamServiceImpl(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService, - dcatServerService, eventSourceService, eventPublisher); - - dataset = new DcatDataset(COLLECTION, readModelFromFile("dcat/dataset/valid.ttl")); - eventStreamTOWithDataset = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, - VERSION_OF_PATH, VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of(), dataset); - } - - @Test - void when_retrieveAllEventStream_then_returnList() { - final String otherCollection = "other"; - EventStream otherEventStream = new EventStream(otherCollection, "created", "versionOf", false); - List views = List - .of(new ViewSpecification(new ViewName("other", "view1"), List.of(), List.of(), 100)); - - EventStreamTO otherEventStreamTO = new EventStreamTO(otherCollection, "created", "versionOf", false, - views, ModelFactory.createDefaultModel(), List.of(), dataset); - - when(eventStreamRepository.retrieveAllEventStreams()).thenReturn(List.of(EVENT_STREAM, otherEventStream)); - when(viewService.getViewsByCollectionName(otherCollection)).thenReturn(views); - when(viewService.getViewsByCollectionName(COLLECTION)).thenReturn(List.of()); - - when(shaclShapeService.retrieveShaclShape(COLLECTION)) - .thenReturn(new ShaclShape(COLLECTION, ModelFactory.createDefaultModel())); - when(shaclShapeService.retrieveShaclShape(otherCollection)) - .thenReturn(new ShaclShape(otherCollection, ModelFactory.createDefaultModel())); - - when(dcatDatasetService.retrieveDataset(COLLECTION)).thenReturn(Optional.empty()); - when(dcatDatasetService.retrieveDataset(otherCollection)).thenReturn(Optional.of(dataset)); - - List eventStreams = service.retrieveAllEventStreams(); - - assertThat(eventStreams).containsExactlyInAnyOrder(EVENT_STREAM_RESPONSE, otherEventStreamTO); - InOrder inOrder = inOrder(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService); - inOrder.verify(eventStreamRepository).retrieveAllEventStreams(); - inOrder.verify(viewService).getViewsByCollectionName(COLLECTION); - inOrder.verify(shaclShapeService).retrieveShaclShape(COLLECTION); - inOrder.verify(dcatDatasetService).retrieveDataset(COLLECTION); - inOrder.verify(viewService).getViewsByCollectionName(otherCollection); - inOrder.verify(shaclShapeService).retrieveShaclShape(otherCollection); - inOrder.verify(dcatDatasetService).retrieveDataset(otherCollection); - - } - - @Test - void when_collectionExists_then_retrieveEventStream() { - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); - when(viewService.getViewsByCollectionName(COLLECTION)).thenReturn(List.of()); - when(shaclShapeService.retrieveShaclShape(COLLECTION)).thenReturn( - new ShaclShape(COLLECTION, ModelFactory.createDefaultModel())); - when(dcatDatasetService.retrieveDataset(COLLECTION)).thenReturn(Optional.empty()); - - EventStreamTO eventStreamTO = service.retrieveEventStream(COLLECTION); - - assertThat(eventStreamTO).isEqualTo(EVENT_STREAM_RESPONSE); - InOrder inOrder = inOrder(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService); - inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); - inOrder.verify(viewService).getViewsByCollectionName(COLLECTION); - inOrder.verify(shaclShapeService).retrieveShaclShape(COLLECTION); - inOrder.verify(dcatDatasetService).retrieveDataset(COLLECTION); - } - - @Test - void when_collectionAndDatasetExists_then_retrieveEventStreamWithDataset() { - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); - when(viewService.getViewsByCollectionName(COLLECTION)).thenReturn(List.of()); - when(shaclShapeService.retrieveShaclShape(COLLECTION)).thenReturn( - new ShaclShape(COLLECTION, ModelFactory.createDefaultModel())); - when(dcatDatasetService.retrieveDataset(COLLECTION)).thenReturn(Optional.of(dataset)); - - - EventStreamTO eventStreamTO = service.retrieveEventStream(COLLECTION); - - assertThat(eventStreamTO).isEqualTo(eventStreamTOWithDataset); - InOrder inOrder = inOrder(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService); - inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); - inOrder.verify(viewService).getViewsByCollectionName(COLLECTION); - inOrder.verify(shaclShapeService).retrieveShaclShape(COLLECTION); - inOrder.verify(dcatDatasetService).retrieveDataset(COLLECTION); - } - - @Test - void when_collectionDoesNotExist_and_retrieveCollection_then_throwException() { - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> service.retrieveEventStream(COLLECTION)) - .isInstanceOf(MissingResourceException.class) + private static final String COLLECTION = "collection"; + private static final String TIMESTAMP_PATH = "generatedAt"; + private static final String VERSION_OF_PATH = "isVersionOf"; + private static final boolean VERSION_CREATION_ENABLED = false; + private static final EventStream EVENT_STREAM = new EventStream(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, VERSION_CREATION_ENABLED); + private static final EventStreamTO EVENT_STREAM_RESPONSE = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, + VERSION_OF_PATH, VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of()); + private DcatDataset dataset; + private EventStreamTO eventStreamTOWithDataset; + + @Mock + private EventStreamRepository eventStreamRepository; + @Mock + private ApplicationEventPublisher eventPublisher; + @Captor + ArgumentCaptor deletedEventArgumentCaptor; + @Mock + private ViewService viewService; + @Mock + private ShaclShapeService shaclShapeService; + @Mock + private DcatDatasetService dcatDatasetService; + @Mock + private DcatServerService dcatServerService; + @Mock + private EventSourceService eventSourceService; + + private EventStreamService service; + + @BeforeEach + void setUp() throws URISyntaxException { + service = new EventStreamServiceImpl(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService, + dcatServerService, eventSourceService, eventPublisher); + + dataset = new DcatDataset(COLLECTION, readModelFromFile("dcat/dataset/valid.ttl")); + eventStreamTOWithDataset = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, + VERSION_OF_PATH, VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of(), dataset); + } + + @Test + void when_retrieveAllEventStream_then_returnList() { + final String otherCollection = "other"; + EventStream otherEventStream = new EventStream(otherCollection, "created", "versionOf", false); + List views = List + .of(new ViewSpecification(new ViewName("other", "view1"), List.of(), List.of(), 100)); + + EventStreamTO otherEventStreamTO = new EventStreamTO(otherCollection, "created", "versionOf", false, + views, ModelFactory.createDefaultModel(), List.of(), dataset); + + when(eventStreamRepository.retrieveAllEventStreams()).thenReturn(List.of(EVENT_STREAM, otherEventStream)); + when(viewService.getViewsByCollectionName(otherCollection)).thenReturn(views); + when(viewService.getViewsByCollectionName(COLLECTION)).thenReturn(List.of()); + + when(shaclShapeService.retrieveShaclShape(COLLECTION)) + .thenReturn(new ShaclShape(COLLECTION, ModelFactory.createDefaultModel())); + when(shaclShapeService.retrieveShaclShape(otherCollection)) + .thenReturn(new ShaclShape(otherCollection, ModelFactory.createDefaultModel())); + + when(dcatDatasetService.retrieveDataset(COLLECTION)).thenReturn(Optional.empty()); + when(dcatDatasetService.retrieveDataset(otherCollection)).thenReturn(Optional.of(dataset)); + + List eventStreams = service.retrieveAllEventStreams(); + + assertThat(eventStreams).containsExactlyInAnyOrder(EVENT_STREAM_RESPONSE, otherEventStreamTO); + InOrder inOrder = inOrder(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService); + inOrder.verify(eventStreamRepository).retrieveAllEventStreams(); + inOrder.verify(viewService).getViewsByCollectionName(COLLECTION); + inOrder.verify(shaclShapeService).retrieveShaclShape(COLLECTION); + inOrder.verify(dcatDatasetService).retrieveDataset(COLLECTION); + inOrder.verify(viewService).getViewsByCollectionName(otherCollection); + inOrder.verify(shaclShapeService).retrieveShaclShape(otherCollection); + inOrder.verify(dcatDatasetService).retrieveDataset(otherCollection); + + } + + @Test + void when_collectionExists_then_retrieveEventStream() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); + when(viewService.getViewsByCollectionName(COLLECTION)).thenReturn(List.of()); + when(shaclShapeService.retrieveShaclShape(COLLECTION)).thenReturn( + new ShaclShape(COLLECTION, ModelFactory.createDefaultModel())); + when(dcatDatasetService.retrieveDataset(COLLECTION)).thenReturn(Optional.empty()); + + EventStreamTO eventStreamTO = service.retrieveEventStream(COLLECTION); + + assertThat(eventStreamTO).isEqualTo(EVENT_STREAM_RESPONSE); + InOrder inOrder = inOrder(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService); + inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); + inOrder.verify(viewService).getViewsByCollectionName(COLLECTION); + inOrder.verify(shaclShapeService).retrieveShaclShape(COLLECTION); + inOrder.verify(dcatDatasetService).retrieveDataset(COLLECTION); + } + + @Test + void when_collectionAndDatasetExists_then_retrieveEventStreamWithDataset() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); + when(viewService.getViewsByCollectionName(COLLECTION)).thenReturn(List.of()); + when(shaclShapeService.retrieveShaclShape(COLLECTION)).thenReturn( + new ShaclShape(COLLECTION, ModelFactory.createDefaultModel())); + when(dcatDatasetService.retrieveDataset(COLLECTION)).thenReturn(Optional.of(dataset)); + + + EventStreamTO eventStreamTO = service.retrieveEventStream(COLLECTION); + + assertThat(eventStreamTO).isEqualTo(eventStreamTOWithDataset); + InOrder inOrder = inOrder(eventStreamRepository, viewService, shaclShapeService, dcatDatasetService); + inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); + inOrder.verify(viewService).getViewsByCollectionName(COLLECTION); + inOrder.verify(shaclShapeService).retrieveShaclShape(COLLECTION); + inOrder.verify(dcatDatasetService).retrieveDataset(COLLECTION); + } + + @Test + void when_collectionDoesNotExist_and_retrieveCollection_then_throwException() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); + + assertThatThrownBy(() -> service.retrieveEventStream(COLLECTION)) + .isInstanceOf(MissingResourceException.class) + .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); + + verify(eventStreamRepository).retrieveEventStream(COLLECTION); + verifyNoInteractions(viewService, shaclShapeService); + } + + @Test + void when_collectionExists_then_updateEventSource() { + service.updateEventSource(COLLECTION, List.of()); + + InOrder inOrder = inOrder(eventSourceService, eventPublisher); + inOrder.verify(eventSourceService).saveEventSource(COLLECTION, List.of()); + } + + @Nested + class CreateEventStream { + private static final String TIMESTAMP_PATH = "generatedAt"; + private static final String VERSION_OF_PATH = "versionOf"; + private static final EventStream EVENT_STREAM = new EventStream(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, VERSION_CREATION_ENABLED); + + @Test + void given_NonExistingEventStream_when_createEventStream_then_expectCreatedEventStream() { + ShaclShape shaclShape = new ShaclShape(COLLECTION, ModelFactory.createDefaultModel()); + when(eventStreamRepository.saveEventStream(EVENT_STREAM)).thenReturn(EVENT_STREAM); + when(shaclShapeService.updateShaclShape(shaclShape)).thenReturn(shaclShape); + EventStreamTO eventStreamTO = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, + VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of()); + + EventStreamTO createdEventStream = service.createEventStream(eventStreamTO); + + assertThat(createdEventStream).isEqualTo(eventStreamTO); + InOrder inOrder = inOrder(eventStreamRepository, shaclShapeService, viewService); + inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); + inOrder.verify(eventStreamRepository).saveEventStream(EVENT_STREAM); + inOrder.verify(shaclShapeService).updateShaclShape(shaclShape); + } + + @Test + void given_ExistingEventStream_when_createEventStreamWithSameName_then_throwException() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); + EventStreamTO eventStreamTO = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, + VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of()); + + assertThatThrownBy(() -> service.createEventStream(eventStreamTO)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("This collection already exists!"); + + InOrder inOrder = inOrder(eventStreamRepository, shaclShapeService, viewService); + inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); + inOrder.verifyNoMoreInteractions(); + } + + @Test + void given_NonExistingEventStream_when_errorOccursWhileCreation_then_DeleteAgainAndThrowException() { + final String byPage = "by-page"; + final String byLocation = "by-location"; + ShaclShape shaclShape = new ShaclShape(COLLECTION, ModelFactory.createDefaultModel()); + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); + ViewSpecification byPageView = new ViewSpecification(new ViewName(COLLECTION, byPage), List.of(), List.of(), 100); + ViewSpecification byLocationView = new ViewSpecification(new ViewName(COLLECTION, byLocation), List.of(), List.of(), 100); + EventStreamTO eventStreamTO = new EventStreamTO( + COLLECTION, + TIMESTAMP_PATH, + VERSION_OF_PATH, + VERSION_CREATION_ENABLED, + List.of(byPageView, byLocationView), + ModelFactory.createDefaultModel(), + List.of()); + + + doNothing().when(viewService).addView(byPageView); + doThrow(new DuplicateRetentionException()).when(viewService).addView(byLocationView); + + assertThatThrownBy(() -> service.createEventStream(eventStreamTO)) + .isInstanceOf(DuplicateRetentionException.class); + InOrder inOrder = inOrder(eventStreamRepository, shaclShapeService, viewService, eventPublisher); + inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); + inOrder.verify(shaclShapeService).updateShaclShape(shaclShape); + inOrder.verify(viewService).addView(byPageView); + inOrder.verify(viewService).addView(byLocationView); + inOrder.verify(eventStreamRepository).deleteEventStream(COLLECTION); + inOrder.verify(eventPublisher).publishEvent(deletedEventArgumentCaptor.capture()); + assertThat(deletedEventArgumentCaptor.getValue()).isEqualTo(new EventStreamDeletedEvent(COLLECTION)); + } + } + + + @Test + void when_collectionDoesNotExists_and_triesToDelete_then_throwException() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); + + assertThatThrownBy(() -> service.deleteEventStream(COLLECTION)) + .isInstanceOf(MissingResourceException.class) + .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); + + verify(eventStreamRepository).retrieveEventStream(COLLECTION); + verifyNoMoreInteractions(eventStreamRepository); + verifyNoInteractions(viewService, shaclShapeService, eventPublisher); + } + + @Test + void when_collectionExists_and_triesToDeleteEventStream_then_throwExceptionWithRetrieval() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)) + .thenReturn(Optional.empty()); + + service.deleteEventStream(COLLECTION); + + InOrder inOrder = inOrder(eventStreamRepository, eventPublisher); + inOrder.verify(eventStreamRepository).deleteEventStream(COLLECTION); + inOrder.verify(eventPublisher).publishEvent(deletedEventArgumentCaptor.capture()); + assertThat(deletedEventArgumentCaptor.getValue()).isEqualTo(new EventStreamDeletedEvent(COLLECTION)); + assertThatThrownBy(() -> service.retrieveEventStream(COLLECTION)) + .isInstanceOf(MissingResourceException.class) + .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); + } + + @Test + void when_init() { + ((EventStreamServiceImpl) service).initEventStream(); + verify(eventStreamRepository).retrieveAllEventStreams(); + } + + @Test + void should_CallDcatServiceToGetDcat_when_GetComposedDcatIsCalled() { + service.getComposedDcat(); + + verify(dcatServerService).getComposedDcat(); + verifyNoMoreInteractions(dcatServerService); + } + + @Test + void when_closeEventStream_thenPublishEventStreamClosedEvent() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); + + service.closeEventStream(COLLECTION); + + verify(eventStreamRepository).retrieveEventStream(COLLECTION); + verify(eventPublisher).publishEvent(new EventStreamClosedEvent(COLLECTION)); + } + + @Test + void when_closeEventStream_andEventStreamDoesNotExist_thenExceptionIsThrown() { + when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); + + assertThatThrownBy(() -> service.closeEventStream(COLLECTION)) + .isInstanceOf(MissingResourceException.class) .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); - verify(eventStreamRepository).retrieveEventStream(COLLECTION); - verifyNoInteractions(viewService, shaclShapeService); - } - - @Test - void when_collectionExists_then_updateEventSource() { - service.updateEventSource(COLLECTION, List.of()); - - InOrder inOrder = inOrder(eventSourceService, eventPublisher); - inOrder.verify(eventSourceService).saveEventSource(COLLECTION, List.of()); - } - - @Nested - class CreateEventStream { - private static final String TIMESTAMP_PATH = "generatedAt"; - private static final String VERSION_OF_PATH = "versionOf"; - private static final EventStream EVENT_STREAM = new EventStream(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, VERSION_CREATION_ENABLED); - - @Test - void given_NonExistingEventStream_when_createEventStream_then_expectCreatedEventStream() { - ShaclShape shaclShape = new ShaclShape(COLLECTION, ModelFactory.createDefaultModel()); - when(eventStreamRepository.saveEventStream(EVENT_STREAM)).thenReturn(EVENT_STREAM); - when(shaclShapeService.updateShaclShape(shaclShape)).thenReturn(shaclShape); - EventStreamTO eventStreamTO = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, - VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of()); - - EventStreamTO createdEventStream = service.createEventStream(eventStreamTO); - - assertThat(createdEventStream).isEqualTo(eventStreamTO); - InOrder inOrder = inOrder(eventStreamRepository, shaclShapeService, viewService); - inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); - inOrder.verify(eventStreamRepository).saveEventStream(EVENT_STREAM); - inOrder.verify(shaclShapeService).updateShaclShape(shaclShape); - } - - @Test - void given_ExistingEventStream_when_createEventStreamWithSameName_then_throwException() { - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)); - EventStreamTO eventStreamTO = new EventStreamTO(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, - VERSION_CREATION_ENABLED, List.of(), ModelFactory.createDefaultModel(), List.of()); - - assertThatThrownBy(() -> service.createEventStream(eventStreamTO)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("This collection already exists!"); - - InOrder inOrder = inOrder(eventStreamRepository, shaclShapeService, viewService); - inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); - inOrder.verifyNoMoreInteractions(); - } - - @Test - void given_NonExistingEventStream_when_errorOccursWhileCreation_then_DeleteAgainAndThrowException() { - final String byPage = "by-page"; - final String byLocation = "by-location"; - ShaclShape shaclShape = new ShaclShape(COLLECTION, ModelFactory.createDefaultModel()); - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); - ViewSpecification byPageView = new ViewSpecification(new ViewName(COLLECTION, byPage), List.of(), List.of(), 100); - ViewSpecification byLocationView = new ViewSpecification(new ViewName(COLLECTION, byLocation), List.of(), List.of(), 100); - EventStreamTO eventStreamTO = new EventStreamTO( - COLLECTION, - TIMESTAMP_PATH, - VERSION_OF_PATH, - VERSION_CREATION_ENABLED, - List.of(byPageView, byLocationView), - ModelFactory.createDefaultModel(), - List.of()); - - - doNothing().when(viewService).addView(byPageView); - doThrow(new DuplicateRetentionException()).when(viewService).addView(byLocationView); - - assertThatThrownBy(() -> service.createEventStream(eventStreamTO)) - .isInstanceOf(DuplicateRetentionException.class); - InOrder inOrder = inOrder(eventStreamRepository, shaclShapeService, viewService, eventPublisher); - inOrder.verify(eventStreamRepository).retrieveEventStream(COLLECTION); - inOrder.verify(shaclShapeService).updateShaclShape(shaclShape); - inOrder.verify(viewService).addView(byPageView); - inOrder.verify(viewService).addView(byLocationView); - inOrder.verify(eventStreamRepository).deleteEventStream(COLLECTION); - inOrder.verify(eventPublisher).publishEvent(deletedEventArgumentCaptor.capture()); - assertThat(deletedEventArgumentCaptor.getValue()).isEqualTo(new EventStreamDeletedEvent(COLLECTION)); - } - } - - - @Test - void when_collectionDoesNotExists_and_triesToDelete_then_throwException() { - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> service.deleteEventStream(COLLECTION)) - .isInstanceOf(MissingResourceException.class) - .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); - - verify(eventStreamRepository).retrieveEventStream(COLLECTION); - verifyNoMoreInteractions(eventStreamRepository); - verifyNoInteractions(viewService, shaclShapeService, eventPublisher); - } - - @Test - void when_collectionExists_and_triesToDeleteEventStream_then_throwExceptionWithRetrieval() { - when(eventStreamRepository.retrieveEventStream(COLLECTION)).thenReturn(Optional.of(EVENT_STREAM)) - .thenReturn(Optional.empty()); + verify(eventStreamRepository).retrieveEventStream(COLLECTION); + verify(eventPublisher, never()).publishEvent(new EventStreamClosedEvent(COLLECTION)); + } - service.deleteEventStream(COLLECTION); - - InOrder inOrder = inOrder(eventStreamRepository, eventPublisher); - inOrder.verify(eventStreamRepository).deleteEventStream(COLLECTION); - inOrder.verify(eventPublisher).publishEvent(deletedEventArgumentCaptor.capture()); - assertThat(deletedEventArgumentCaptor.getValue()).isEqualTo(new EventStreamDeletedEvent(COLLECTION)); - assertThatThrownBy(() -> service.retrieveEventStream(COLLECTION)) - .isInstanceOf(MissingResourceException.class) - .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); - } - - @Test - void when_init() { - ((EventStreamServiceImpl) service).initEventStream(); - verify(eventStreamRepository).retrieveAllEventStreams(); - } - - @Test - void should_CallDcatServiceToGetDcat_when_GetComposedDcatIsCalled() { - service.getComposedDcat(); - - verify(dcatServerService).getComposedDcat(); - verifyNoMoreInteractions(dcatServerService); - } - - private Model readModelFromFile(String fileName) throws URISyntaxException { - ClassLoader classLoader = getClass().getClassLoader(); - String uri = Objects.requireNonNull(classLoader.getResource(fileName)).toURI() - .toString(); - return RDFDataMgr.loadModel(uri); - } + private Model readModelFromFile(String fileName) throws URISyntaxException { + ClassLoader classLoader = getClass().getClassLoader(); + String uri = Objects.requireNonNull(classLoader.getResource(fileName)).toURI() + .toString(); + return RDFDataMgr.loadModel(uri); + } } diff --git a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java index 086c6e9211..b8a1b46d1a 100644 --- a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java +++ b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java @@ -240,6 +240,28 @@ void when_deleteNotExistingCollection_then_expectStatus404() throws Exception { } } + @Nested + class CloseEventStream { + @Test + void when_collectionExists_and_triesToClose_then_expectStatus200() throws Exception { + mockMvc.perform(put("/admin/api/v1/eventstreams/name1")) + .andExpect(status().isOk()); + + verify(eventStreamService).closeEventStream(COLLECTION); + } + + @Test + void when_collectionDoesNotExist_then_expectStatus404() throws Exception { + doThrow(MissingResourceException.class).when(eventStreamService).closeEventStream(COLLECTION); + + mockMvc.perform(put("/admin/api/v1/eventstreams/name1")) + .andExpect(status().isNotFound()) + .andExpect(content().contentType(MediaType.TEXT_PLAIN)); + + verify(eventStreamService).closeEventStream(COLLECTION); + } + } + private Model readModelFromFile(String fileName) throws URISyntaxException { ClassLoader classLoader = getClass().getClassLoader(); String uri = Objects.requireNonNull(classLoader.getResource(fileName)).toURI() diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/admin/EventStreamClosedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/admin/EventStreamClosedEvent.java new file mode 100644 index 0000000000..756f4a1f2e --- /dev/null +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/admin/EventStreamClosedEvent.java @@ -0,0 +1,4 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin; + +public record EventStreamClosedEvent(String collectionName) { +} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/EventStream.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/EventStream.java index b742704708..0d2e01ba9d 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/EventStream.java +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/EventStream.java @@ -8,12 +8,25 @@ public class EventStream { private final String timestampPath; private final String versionOfPath; private final boolean versionCreationEnabled; + private final boolean isClosed; - public EventStream(String collection, String timestampPath, String versionOfPath, boolean versionCreationEnabled) { + public EventStream(String collection, + String timestampPath, + String versionOfPath, + boolean versionCreationEnabled) { + this(collection, timestampPath, versionOfPath, versionCreationEnabled, false); + } + + public EventStream(String collection, + String timestampPath, + String versionOfPath, + boolean versionCreationEnabled, + boolean isClosed) { this.collection = collection; this.timestampPath = timestampPath; this.versionOfPath = versionOfPath; this.versionCreationEnabled = versionCreationEnabled; + this.isClosed = isClosed; } public String getCollection() { @@ -32,6 +45,10 @@ public boolean isVersionCreationEnabled() { return versionCreationEnabled; } + public boolean isClosed() { + return isClosed; + } + @Override public boolean equals(Object o) { if (this == o) diff --git a/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/entity/EventStreamEntity.java b/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/entity/EventStreamEntity.java index 8fb98e5e84..bd784f06a2 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/entity/EventStreamEntity.java +++ b/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/entity/EventStreamEntity.java @@ -12,14 +12,23 @@ public class EventStreamEntity { private String timestampPath; private String versionOfPath; private boolean versionCreationEnabled; + private boolean isClosed; protected EventStreamEntity() {} - public EventStreamEntity(String id, String timestampPath, String versionOfPath, boolean versionCreationEnabled) { + public EventStreamEntity(String id, + String timestampPath, + String versionOfPath, + boolean versionCreationEnabled) { + this(id, timestampPath, versionOfPath, versionCreationEnabled, false); + } + + public EventStreamEntity(String id, String timestampPath, String versionOfPath, boolean versionCreationEnabled, boolean isClosed) { this.id = id; this.timestampPath = timestampPath; this.versionOfPath = versionOfPath; this.versionCreationEnabled = versionCreationEnabled; + this.isClosed = isClosed; } public String getId() { @@ -37,4 +46,8 @@ public String getVersionOfPath() { public boolean isVersionCreationEnabled() { return versionCreationEnabled; } + + public boolean isClosed() { + return isClosed; + } } diff --git a/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/service/EventStreamConverter.java b/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/service/EventStreamConverter.java index f44a2ab3bb..70abcaf10f 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/service/EventStreamConverter.java +++ b/ldes-server-infra-postgres/postgres-admin-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/postgres/eventstream/service/EventStreamConverter.java @@ -9,11 +9,11 @@ public class EventStreamConverter { public EventStreamEntity fromEventStream(EventStream eventStream) { return new EventStreamEntity(eventStream.getCollection(), eventStream.getTimestampPath(), - eventStream.getVersionOfPath(), eventStream.isVersionCreationEnabled()); + eventStream.getVersionOfPath(), eventStream.isVersionCreationEnabled(), eventStream.isClosed()); } public EventStream toEventStream(EventStreamEntity eventStreamEntity) { return new EventStream(eventStreamEntity.getId(), eventStreamEntity.getTimestampPath(), - eventStreamEntity.getVersionOfPath(), eventStreamEntity.isVersionCreationEnabled()); + eventStreamEntity.getVersionOfPath(), eventStreamEntity.isVersionCreationEnabled(), eventStreamEntity.isClosed()); } } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentPostgresRepository.java index 0d196a3f96..81869a7964 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/FragmentPostgresRepository.java @@ -128,7 +128,14 @@ public void makeChildrenImmutable(Fragment fragment) { log.atInfo().log("{} child/children of {} was/were made immutable.", modifiedRows, fragment.getFragmentIdString()); } - private void removeRelationsPointingToDeletedFragment(LdesFragmentIdentifier readyForDeletionFragmentId) { + @Override + @Transactional + public void markFragmentsImmutableInCollection(String collectionName) { + repository.markFragmentsImmutableInCollection(collectionName); + log.info("All fragments are made immutable in collection {}", collectionName); + } + + private void removeRelationsPointingToDeletedFragment(LdesFragmentIdentifier readyForDeletionFragmentId) { List fragments = repository.findAllByRelations_TreeNode(readyForDeletionFragmentId.asDecodedFragmentId()); fragments.forEach(fragment -> { List relationsToRemove = fragment.getRelations().stream() diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java index a478c46390..13243a4320 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/repository/FragmentEntityRepository.java @@ -51,5 +51,10 @@ public interface FragmentEntityRepository extends JpaRepository findByDeleteTimeNotNull(); + Stream findByDeleteTimeNotNull(); + + @Modifying + @Query(value = "UPDATE fragmentation_fragment SET immutable = true WHERE collection_name = :collectionName", + nativeQuery = true) + void markFragmentsImmutableInCollection(@Param("collectionName") String collectionName); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/fragment/FragmentPostgresRepositoryTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/fragment/FragmentPostgresRepositoryTest.java index 965fbfcec3..349420a321 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/fragment/FragmentPostgresRepositoryTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/fragment/FragmentPostgresRepositoryTest.java @@ -8,6 +8,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.FragmentEntity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.FragmentEntityRepository; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -19,6 +20,9 @@ import java.util.Optional; import java.util.stream.Stream; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + class FragmentPostgresRepositoryTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); @@ -34,7 +38,7 @@ class FragmentPostgresRepositoryTest { @ArgumentsSource(LdesFragmentEntityListProvider.class) void when_RetrieveOpenFragment_FirstFragmentThatIsOpenAndBelongsToCollectionIsReturned( List entitiesInRepository, String expectedFragmentId) { - Mockito.when(ldesFragmentEntityRepository + when(ldesFragmentEntityRepository .findAllByImmutableAndViewName(false, VIEW_NAME.asString())) .thenReturn(entitiesInRepository.stream() @@ -51,6 +55,13 @@ void when_RetrieveOpenFragment_FirstFragmentThatIsOpenAndBelongsToCollectionIsRe Assertions.assertEquals(expectedFragmentId, ldesFragment.get().getFragmentIdString()); } + @Test + void markFragmentsImmutableInCollection() { + ldesFragmentPostgresRepository.markFragmentsImmutableInCollection("collectionName"); + + verify(ldesFragmentEntityRepository).markFragmentsImmutableInCollection("collectionName"); + } + static class LdesFragmentEntityListProvider implements ArgumentsProvider { @Override diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/eventstream-closed-column.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/eventstream-closed-column.xml new file mode 100644 index 0000000000..8e83914dec --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/eventstream-closed-column.xml @@ -0,0 +1,13 @@ + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/master.xml new file mode 100644 index 0000000000..da0df7822b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_1_0/master.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index bd6964cd6d..2b71947ee1 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -3,5 +3,6 @@ + \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java index 45bcdda6df..70d2961f07 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java @@ -20,120 +20,134 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; import static org.apache.jena.rdf.model.ResourceFactory.createResource; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.hamcrest.core.StringContains.containsString; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public class FragmentationSteps extends LdesServerIntegrationTest { private static final String TREE = "https://w3id.org/tree#"; - private static final Property TREE_MEMBER = createProperty(TREE, "member"); private Model currentFragment; private String currentPath; private String currentFragmentCacheControl; - @When("I fetch the root {string} fragment of {string}") - public void iFetchTheRootFragment(String view, String collection) throws Exception { - currentPath = "/%s/%s".formatted(collection, view); - MockHttpServletResponse response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) - .andReturn() - .getResponse(); - currentFragmentCacheControl = response.getHeader("Cache-Control"); - currentFragment = new ResponseToModelConverter(response).convert(); - } - - private void fetchFragment(String path) throws Exception { - currentPath = path; - MockHttpServletResponse response = mockMvc.perform(get(new URI(path)) - .accept("text/turtle")) - .andReturn() - .getResponse(); - currentFragmentCacheControl = response.getHeader("Cache-Control"); - currentFragment = new ResponseToModelConverter(response).convert(); - } - - @And("I fetch the next fragment through the first {string}") - public void iFetchTheNextFragmentThroughTheFirst(String relation) throws Exception { - Resource relationSubj = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) - .next().getSubject(); - - currentPath = currentFragment.listStatements(relationSubj, createProperty(TREE, "node"), (Resource) null) - .next().getObject().toString(); - - - - MockHttpServletResponse response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) - .andReturn() - .getResponse(); - currentFragmentCacheControl = response.getHeader("Cache-Control"); - currentFragment = new ResponseToModelConverter(response).convert(); - } - - @Then("this fragment only has {int} {string} relation") - public void thisFragmentOnlyHasOne(int expectedRelationCount, String relation) { - await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { - fetchFragment(currentPath); - int relationCount = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) - .toList().size(); - System.out.println(currentPath); - System.out.println("relationcounts: " + relationCount); - return relationCount == expectedRelationCount; - }); - } - - @And("this fragment is immutable") - public void thisFragmentIsImmutable() { - assertTrue(currentFragmentCacheControl.contains("immutable")); - } - - @And("this fragment contains {int} members") - public void thisFragmentContainsMembers(int expectedMemberCount) { - await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { - fetchFragment(currentPath); - return MemberCounter.countMembers(expectedMemberCount).matches(currentFragment); - }); - } - - @And("this fragment is mutable") - public void thisFragmentIsNotImmutable() { - assertFalse(currentFragmentCacheControl.contains("immutable")); - } - - @And("this fragment has no relations") - public void thisFragmentHasNoRelations() { - await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { - fetchFragment(currentPath); - return !currentFragment.listObjectsOfProperty(createProperty(TREE + "relation")).hasNext(); - }); - } - - @When("I fetch the {string} fragment for {string} from the {string} view of {string}") - public void iFetchTheFragmentOf(String fragmentKey, String fragmentValue, String view, String collection) - throws Exception { - currentPath = "/%s/%s?%s=%s".formatted(collection, view, fragmentKey, fragmentValue); - iFetchTheFragmentOf(currentPath); - } - - @When("I fetch the {string} fragment") - public void iFetchTheFragmentOf(String path) - throws Exception { - currentPath = path; - MockHttpServletResponse response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) - .andReturn() - .getResponse(); - currentFragmentCacheControl = response.getHeader("Cache-Control"); - currentFragment = new ResponseToModelConverter(response).convert(); - } - - @When("I fetch the timebased fragment {string} fragment of this month of {string}") - public void iFetchTheTimebasedFragmentFragmentOfTodayOf(String view, String collection) throws Exception { - LocalDateTime now = LocalDateTime.now(); - currentPath = "/%s/%s?year=%s&month=%02d".formatted(collection, view, now.getYear(), now.getMonthValue()); - String response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) - .andReturn() - .getResponse() - .getContentAsString(); - - currentFragment = RDFParser.fromString(response).lang(Lang.TURTLE).toModel(); - } + @When("I fetch the root {string} fragment of {string}") + public void iFetchTheRootFragment(String view, String collection) throws Exception { + currentPath = "/%s/%s".formatted(collection, view); + MockHttpServletResponse response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) + .andReturn() + .getResponse(); + currentFragmentCacheControl = response.getHeader("Cache-Control"); + currentFragment = new ResponseToModelConverter(response).convert(); + } + + private void fetchFragment(String path) throws Exception { + currentPath = path; + MockHttpServletResponse response = mockMvc.perform(get(new URI(path)) + .accept("text/turtle")) + .andReturn() + .getResponse(); + currentFragmentCacheControl = response.getHeader("Cache-Control"); + currentFragment = new ResponseToModelConverter(response).convert(); + } + + @And("I fetch the next fragment through the first {string}") + public void iFetchTheNextFragmentThroughTheFirst(String relation) throws Exception { + Resource relationSubj = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) + .next().getSubject(); + + currentPath = currentFragment.listStatements(relationSubj, createProperty(TREE, "node"), (Resource) null) + .next().getObject().toString(); + + + MockHttpServletResponse response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) + .andReturn() + .getResponse(); + currentFragmentCacheControl = response.getHeader("Cache-Control"); + currentFragment = new ResponseToModelConverter(response).convert(); + } + + @Then("this fragment only has {int} {string} relation") + public void thisFragmentOnlyHasOne(int expectedRelationCount, String relation) { + await().atMost(Duration.of(40, ChronoUnit.SECONDS)).until(() -> { + fetchFragment(currentPath); + int relationCount = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) + .toList().size(); + System.out.println(currentPath); + System.out.println("relationcounts: " + relationCount); + return relationCount == expectedRelationCount; + }); + } + + @And("this fragment is immutable") + public void thisFragmentIsImmutable() { + assertThat(currentFragmentCacheControl) + .contains("immutable") + .contains("max-age=31536000"); + } + + @And("this fragment contains {int} members") + public void thisFragmentContainsMembers(int expectedMemberCount) { + await().atMost(Duration.of(20, ChronoUnit.SECONDS)).until(() -> { + fetchFragment(currentPath); + return MemberCounter.countMembers(expectedMemberCount).matches(currentFragment); + }); + } + + @And("this fragment is mutable") + public void thisFragmentIsNotImmutable() { + assertFalse(currentFragmentCacheControl.contains("immutable")); + } + + @And("this fragment has no relations") + public void thisFragmentHasNoRelations() { + await().atMost(Duration.of(20, ChronoUnit.SECONDS)).until(() -> { + fetchFragment(currentPath); + return !currentFragment.listObjectsOfProperty(createProperty(TREE + "relation")).hasNext(); + }); + } + + @When("I fetch the {string} fragment for {string} from the {string} view of {string}") + public void iFetchTheFragmentOf(String fragmentKey, String fragmentValue, String view, String collection) + throws Exception { + currentPath = "/%s/%s?%s=%s".formatted(collection, view, fragmentKey, fragmentValue); + iFetchTheFragmentOf(currentPath); + } + + @When("I fetch the {string} fragment") + public void iFetchTheFragmentOf(String path) + throws Exception { + currentPath = path; + MockHttpServletResponse response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) + .andReturn() + .getResponse(); + currentFragmentCacheControl = response.getHeader("Cache-Control"); + currentFragment = new ResponseToModelConverter(response).convert(); + } + + @When("I fetch the timebased fragment {string} fragment of this month of {string}") + public void iFetchTheTimebasedFragmentFragmentOfTodayOf(String view, String collection) throws Exception { + LocalDateTime now = LocalDateTime.now(); + currentPath = "/%s/%s?year=%s&month=%02d".formatted(collection, view, now.getYear(), now.getMonthValue()); + String response = mockMvc.perform(get(new URI(currentPath)).accept("text/turtle")) + .andReturn() + .getResponse() + .getContentAsString(); + + currentFragment = RDFParser.fromString(response).lang(Lang.TURTLE).toModel(); + } + + @Then("the following fragment URL {string} contains member with ID {string}") + public void theLDESCollectionContainsFragments(String fragment, String memberId) { + await() + .atMost(Duration.ofSeconds(20)) + .pollInterval(Duration.ofSeconds(1)) + .untilAsserted(() -> mockMvc.perform(get(fragment)) + .andExpect(status().is2xxSuccessful()) + .andExpect(content().string(containsString(memberId)))); + + } } diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java index 30b25ee55d..4818eb2abd 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java @@ -341,4 +341,10 @@ public void iFetchAStreamingFragment(String url) { public void modelIsIsomorphic(String url) throws Exception { assertTrue(responseModel.isIsomorphicWith(getResponseAsModel(url, Lang.TURTLE.getHeaderString()))); } + + @When("I close the collection {string}") + public void iCloseTheEventstream(String collection) throws Exception { + mockMvc.perform(put("/admin/api/v1/eventstreams/{collection}", collection)) + .andExpect(status().is2xxSuccessful()); + } } diff --git a/ldes-server-integration-test/src/test/resources/features/fragmentation/fragmentation.feature b/ldes-server-integration-test/src/test/resources/features/fragmentation/fragmentation.feature index f3e06e5e8a..4a67450b68 100644 --- a/ldes-server-integration-test/src/test/resources/features/fragmentation/fragmentation.feature +++ b/ldes-server-integration-test/src/test/resources/features/fragmentation/fragmentation.feature @@ -126,4 +126,20 @@ Feature: LDES Server Fragmentation Examples: | eventStreamDescriptionFile | template | collection | | "data/input/eventstreams/fragmentation/mobility-hindrances.by-loc.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | - | "data/input/eventstreams/fragmentation/observations/by-loc.ttl" | "data/input/members/two-observations.template.ttl" | "observations" | \ No newline at end of file + | "data/input/eventstreams/fragmentation/observations/by-loc.ttl" | "data/input/members/two-observations.template.ttl" | "observations" | + + Scenario Outline: Server can close an event stream + Given I create the eventstream + When I ingest 317 members of template