diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml index 375fbcc4fe..496386d180 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.3.0 + 3.4.0-SNAPSHOT 4.0.0 @@ -19,12 +19,6 @@ ${project.version} - - be.vlaanderen.informatievlaanderen.vsds - ldes-server-retention - ${project.version} - - org.springframework.boot diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java index e1c5199d3f..9793c7de77 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java @@ -1,12 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import io.micrometer.observation.Observation; -import java.util.List; - public interface FragmentationStrategy { - List addMemberToBucket(Bucket rootFragmentOfView, FragmentationMember member, Observation parentObservation); + void addMemberToBucket(Bucket rootBucketOfView, FragmentationMember member, Observation parentObservation); } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java index cbc68266af..d589f75a48 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyCreator; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import io.micrometer.observation.ObservationRegistry; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; @@ -16,16 +15,13 @@ @Component public class FragmentationStrategyBatchCollection implements FragmentationStrategyCollection { - private final BucketRepository bucketRepository; private final Set fragmentationStrategySet; private final FragmentationStrategyCreator fragmentationStrategyCreator; private final ObservationRegistry observationRegistry; public FragmentationStrategyBatchCollection( - BucketRepository bucketRepository, FragmentationStrategyCreator fragmentationStrategyCreator, ObservationRegistry observationRegistry) { - this.bucketRepository = bucketRepository; this.fragmentationStrategyCreator = fragmentationStrategyCreator; this.observationRegistry = observationRegistry; this.fragmentationStrategySet = new HashSet<>(); @@ -78,7 +74,6 @@ private void removeFromStrategySet(Predicate private FragmentationStrategyBatchExecutor createExecutor(ViewName viewName, ViewSpecification viewSpecification) { final FragmentationStrategy fragmentationStrategy = fragmentationStrategyCreator .createFragmentationStrategyForView(viewSpecification); - final var rootBucketRetriever = new RootBucketRetriever(viewName, bucketRepository, observationRegistry); - return new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, rootBucketRetriever, observationRegistry); + return new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, observationRegistry); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java index b1bdc4c39e..73c64efe0a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java @@ -1,11 +1,11 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; +import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import java.util.List; import java.util.Objects; import static io.micrometer.observation.Observation.createNotStarted; @@ -14,26 +14,23 @@ public class FragmentationStrategyBatchExecutor { private final FragmentationStrategy fragmentationStrategy; private final ViewName viewName; - private final RootBucketRetriever rootBucketRetriever; private final ObservationRegistry observationRegistry; @SuppressWarnings("java:S107") public FragmentationStrategyBatchExecutor(ViewName viewName, FragmentationStrategy fragmentationStrategy, - RootBucketRetriever rootBucketRetriever, ObservationRegistry observationRegistry) { - this.rootBucketRetriever = rootBucketRetriever; this.observationRegistry = observationRegistry; this.fragmentationStrategy = fragmentationStrategy; this.viewName = viewName; } - public List bucketise(FragmentationMember member) { - var parentObservation = createNotStarted("execute fragmentation", observationRegistry).start(); - final var rootBucket = rootBucketRetriever.retrieveRootBucket(parentObservation); - List bucketisedMembers = fragmentationStrategy.addMemberToBucket(rootBucket, member, parentObservation); + public Bucket bucketise(FragmentationMember member) { + final Observation parentObservation = createNotStarted("execute fragmentation", observationRegistry).start(); + final Bucket rootBucket = Bucket.createRootBucketForView(viewName); + fragmentationStrategy.addMemberToBucket(rootBucket, member, parentObservation); parentObservation.stop(); - return bucketisedMembers; + return rootBucket; } public boolean isPartOfCollection(String collectionName) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java index ce58eb633f..1c2d7a56e1 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java @@ -1,35 +1,19 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; import io.micrometer.observation.Observation; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.List; public abstract class FragmentationStrategyDecorator implements FragmentationStrategy { - private final FragmentationStrategy fragmentationStrategy; - private final ApplicationEventPublisher applicationEventPublisher; - - protected FragmentationStrategyDecorator(FragmentationStrategy fragmentationStrategy, - ApplicationEventPublisher applicationEventPublisher) { + protected FragmentationStrategyDecorator(FragmentationStrategy fragmentationStrategy) { this.fragmentationStrategy = fragmentationStrategy; - this.applicationEventPublisher = applicationEventPublisher; } @Override - public List addMemberToBucket(Bucket rootFragmentOfView, FragmentationMember member, Observation parentObservation) { - return fragmentationStrategy.addMemberToBucket(rootFragmentOfView, member, parentObservation); - } - - protected void addRelationFromParentToChild(Bucket parentBucket, Bucket childBucket) { - BucketRelation bucketRelation = BucketRelation.createGenericRelation(parentBucket, childBucket); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { + fragmentationStrategy.addMemberToBucket(parentBucket, member, parentObservation); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java index e854d75339..d3be5d5dc2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java @@ -1,15 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import io.micrometer.observation.Observation; -import java.util.List; - public class FragmentationStrategyImpl implements FragmentationStrategy { @Override - public List addMemberToBucket(Bucket bucket, FragmentationMember member, Observation parentObservation) { - return List.of(new BucketisedMember(bucket.getBucketId(), member.getMemberId())); + public void addMemberToBucket(Bucket rootBucketOfView, FragmentationMember member, Observation parentObservation) { + rootBucketOfView.assignMember(member.getMemberId()); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java deleted file mode 100644 index f0d7386626..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java +++ /dev/null @@ -1,36 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingRootFragmentException; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; - -public class RootBucketRetriever { - private final ViewName viewName; - private final BucketRepository bucketRepository; - private final ObservationRegistry observationRegistry; - private Bucket rootBucket; - - public RootBucketRetriever(ViewName viewName, BucketRepository bucketRepository, ObservationRegistry observationRegistry) { - this.viewName = viewName; - this.bucketRepository = bucketRepository; - this.observationRegistry = observationRegistry; - } - - public Bucket retrieveRootBucket(Observation parentObservation) { - final Observation rootRetrievalObservation = Observation - .createNotStarted("retrieve root of view %s".formatted(viewName.asString()), observationRegistry) - .parentObservation(parentObservation) - .start(); - - if(rootBucket == null) { - rootBucket = bucketRepository.retrieveRootBucket(viewName) - .orElseThrow(() -> new MissingRootFragmentException(viewName.asString())); - } - - rootRetrievalObservation.stop(); - return rootBucket; - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java index 6e90ae92e2..d3f706f925 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import org.springframework.batch.core.Step; import org.springframework.batch.core.repository.JobRepository; @@ -8,15 +8,10 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.task.SimpleAsyncTaskExecutor; -import org.springframework.core.task.TaskExecutor; import org.springframework.transaction.PlatformTransactionManager; -import java.util.List; - @Configuration public class BucketJobDefinitions { public static final String BUCKETISATION_STEP = "bucketisation"; @@ -26,23 +21,16 @@ public class BucketJobDefinitions { public Step bucketiseMembersStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, ItemReader memberReader, - ItemProcessor> viewBucketProcessor, - ItemWriter> writer, - BucketMetricUpdater bucketMetricUpdater, - @Qualifier("bucketTaskExecutor") TaskExecutor taskExecutor) { + ItemProcessor bucketProcessor, + ItemWriter bucketisationItemWriter, + BucketMetricUpdater bucketMetricUpdater) { return new StepBuilder(BUCKETISATION_STEP, jobRepository) - .>chunk(CHUNK_SIZE, transactionManager) + .chunk(CHUNK_SIZE, transactionManager) .reader(memberReader) - .processor(viewBucketProcessor) - .writer(writer) + .processor(bucketProcessor) + .writer(bucketisationItemWriter) .listener(bucketMetricUpdater) .build(); } - @Bean("bucketTaskExecutor") - public TaskExecutor paginationTaskExecutor() { - var taskExecutor = new SimpleAsyncTaskExecutor("spring_batch"); - taskExecutor.setConcurrencyLimit(1); - return taskExecutor; - } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java index 4b86012728..ee7a5b9827 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.item.ItemProcessor; @@ -10,13 +10,11 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.List; - @Configuration public class BucketProcessors { @Bean @StepScope - public ItemProcessor> viewBucketProcessor( + public ItemProcessor bucketProcessor( FragmentationStrategyCollection fragmentationStrategyCollection, @Value("#{jobParameters['collectionName']}") String collectionName, @Value("#{jobParameters['viewName']}") String viewName diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java index 9b0aafdc88..9bdabe0890 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java @@ -5,25 +5,29 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.DuplicateFragmentPairException; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; +import java.util.*; +import java.util.stream.Stream; public class Bucket { - private final long bucketId; + private long bucketId; private final BucketDescriptor bucketDescriptor; private final ViewName viewName; + private final List children; + private long assignedMemberId; - public Bucket(long bucketId, BucketDescriptor bucketDescriptor, ViewName viewName) { + public Bucket(long bucketId, BucketDescriptor bucketDescriptor, ViewName viewName, List children, long assignedMemberId) { this.bucketId = bucketId; this.bucketDescriptor = bucketDescriptor; this.viewName = viewName; + this.children = new ArrayList<>(children); + this.assignedMemberId = assignedMemberId; } public Bucket(BucketDescriptor bucketDescriptor, ViewName viewName) { - this(0, bucketDescriptor, viewName); + this(0, bucketDescriptor, viewName, new ArrayList<>(), 0); } public static Bucket createRootBucketForView(ViewName viewName) { @@ -34,12 +38,12 @@ public long getBucketId() { return bucketId; } - public ViewName getViewName() { - return viewName; + public void setBucketId(long bucketId) { + this.bucketId = bucketId; } - public List getBucketDescriptorPairs() { - return bucketDescriptor.getDescriptorPairs(); + public ViewName getViewName() { + return viewName; } public BucketDescriptor getBucketDescriptor() { @@ -50,6 +54,26 @@ public String getBucketDescriptorAsString() { return bucketDescriptor.asDecodedString(); } + public ChildBucket addChildBucket(ChildBucket childBucket) { + final int index = children.indexOf(childBucket); + if (index == -1) { + children.add(childBucket); + return childBucket; + } + ChildBucket existingChildBucket = children.get(index); + existingChildBucket.addRelations(childBucket.getRelations()); + return existingChildBucket; + } + + + public ChildBucket withRelations(TreeRelation... relationDefinition) { + return new ChildBucket(bucketId, bucketDescriptor, viewName, children, assignedMemberId, Set.of(relationDefinition)); + } + + public ChildBucket withGenericRelation() { + return withRelations(TreeRelation.generic()); + } + public Bucket createChild(BucketDescriptorPair descriptorPair) { return new Bucket(createChildDescriptor(descriptorPair), viewName); } @@ -63,6 +87,40 @@ public BucketDescriptor createChildDescriptor(BucketDescriptorPair descriptorPai return new BucketDescriptor(childFragmentPairs); } + public void assignMember(long memberId) { + assignedMemberId = memberId; + } + + public Optional getMember() { + return Optional.of(assignedMemberId) + .filter(member -> member != 0) + .map(member -> new BucketisedMember(bucketId, member)); + } + + public List getChildren() { + return List.copyOf(children); + } + + public List getAllDescendants() { + return children.stream() + .flatMap(child -> Stream.concat(Stream.of(child), child.getAllDescendants().stream())) + .toList(); + } + + public List getBucketTree() { + final List bucketTree = new ArrayList<>(List.of(this)); + getAllDescendants().stream().distinct().forEach(bucketTree::add); + return bucketTree; + } + + public List getChildRelations() { + return children.stream() + .flatMap(child -> child.getRelations().stream() + .map(relation -> new BucketRelation(createPartialUrl(), child.createPartialUrl(), relation)) + ) + .toList(); + } + public String createPartialUrl() { return "/" + viewName.asString() + (bucketDescriptor.isEmpty() ? "" : "?" + bucketDescriptor.asDecodedString()); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java new file mode 100644 index 0000000000..6b75a60161 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java @@ -0,0 +1,26 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ChildBucket extends Bucket { + private final Set relations; + + public ChildBucket(long bucketId, BucketDescriptor bucketDescriptor, ViewName viewName, List children, long assignedMemberId, Set relations) { + super(bucketId, bucketDescriptor, viewName, children, assignedMemberId); + this.relations = new HashSet<>(relations); + } + + public Set getRelations() { + return relations; + } + + public void addRelations(Set relations) { + this.relations.addAll(relations); + } +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java deleted file mode 100644 index 9aa76a9928..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java +++ /dev/null @@ -1,164 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.DuplicateFragmentPairException; -import org.jetbrains.annotations.Nullable; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -public class Fragment { - - public static final String ROOT = "root"; - private final LdesFragmentIdentifier identifier; - private Boolean immutable; - private int nrOfMembersAdded; - private final List relations; - private LocalDateTime deleteTime; - private LocalDateTime nextUpdateTs = null; - - public Fragment(LdesFragmentIdentifier identifier) { - this(identifier, false, 0, new ArrayList<>(), null); - } - - public Fragment(LdesFragmentIdentifier identifier, Boolean immutable, int nrOfMembersAdded, - List relations, LocalDateTime deleteTime) { - this.identifier = identifier; - this.immutable = immutable; - this.nrOfMembersAdded = nrOfMembersAdded; - this.relations = relations; - this.deleteTime = deleteTime; - } - - public LdesFragmentIdentifier getFragmentId() { - return identifier; - } - - public String getFragmentIdString() { - return identifier.asDecodedFragmentId(); - } - - public List getFragmentPairs() { - return this.identifier.getFragmentPairs(); - } - - public void makeImmutable() { - this.immutable = true; - } - - public boolean isImmutable() { - return this.immutable; - } - - public Fragment createChild(FragmentPair fragmentPair) { - List childFragmentPairs = new ArrayList<>(this.identifier.getFragmentPairs().stream().toList()); - if (hasChildWithSameFragmentKey(fragmentPair, childFragmentPairs)) { - throw new DuplicateFragmentPairException(identifier.asDecodedFragmentId(), fragmentPair.fragmentKey()); - } - childFragmentPairs.add(fragmentPair); - return new Fragment(new LdesFragmentIdentifier(getViewName(), childFragmentPairs)); - } - - private static boolean hasChildWithSameFragmentKey(FragmentPair fragmentPair, List childFragmentPairs) { - return childFragmentPairs - .stream() - .map(FragmentPair::fragmentKey) - .anyMatch(key -> key.equals(fragmentPair.fragmentKey())); - } - - public Optional getValueOfKey(String key) { - return this.identifier.getValueOfFragmentPairKey(key); - } - - public ViewName getViewName() { - return this.identifier.getViewName(); - } - - public int getNrOfMembersAdded() { - return this.nrOfMembersAdded; - } - - public int incrementNrOfMembersAdded() { - return this.nrOfMembersAdded++; - } - - public Optional getParentId() { - return identifier.getParentId(); - } - - public String getParentIdAsString() { - return identifier.getParentId().map(LdesFragmentIdentifier::asDecodedFragmentId).orElse(ROOT); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - Fragment that = (Fragment) o; - return Objects.equals(getFragmentId(), that.getFragmentId()); - } - - @Override - public int hashCode() { - return Objects.hash(getFragmentId()); - } - - public void addRelation(TreeRelation relation) { - relations.add(relation); - } - - public boolean containsRelation(TreeRelation parentChildRelation) { - return relations.contains(parentChildRelation); - } - - public List getRelations() { - return relations; - } - - public boolean isRoot() { - return this.identifier.getFragmentPairs().isEmpty(); - } - - public LocalDateTime getDeleteTime() { - return deleteTime; - } - - public void removeRelationToIdentifier(LdesFragmentIdentifier fragmentIdentifier) { - relations.removeIf(treeRelation -> treeRelation.treeNode().equals(fragmentIdentifier)); - } - - public boolean isConnectedTo(Fragment otherFragment) { - return getRelations() - .stream() - .anyMatch(treeRelation -> treeRelation.treeNode() - .equals(otherFragment.getFragmentId())); - } - - @Nullable - public LocalDateTime getNextUpdateTs() { - return nextUpdateTs; - } - - public void setNextUpdateTs(@Nullable LocalDateTime nextUpdateTs) { - this.nextUpdateTs = nextUpdateTs; - } - - @Override - public String toString() { - return "Fragment{" + - "identifier=" + identifier + - ", immutable=" + immutable + - ", nrOfMembersAdded=" + nrOfMembersAdded + - ", relations=" + relations + - ", deleteTime=" + deleteTime + - '}'; - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java deleted file mode 100644 index f9f394780c..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java +++ /dev/null @@ -1,10 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; - -public interface RootFragmentCreator { - - Fragment createRootFragmentForView(ViewName viewName); - -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java index 8fa3978003..b2adaec89e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java @@ -2,14 +2,10 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import java.util.Optional; public interface BucketRepository { - Optional retrieveBucket(ViewName viewName, BucketDescriptor bucketDescriptor); - Bucket insertBucket(Bucket bucket); - Optional retrieveRootBucket(ViewName viewName); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java deleted file mode 100644 index eaf60df278..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -public interface BucketisedMemberRepository { - void deleteByViewName(ViewName viewName); - void deleteByCollection(String collectionName); -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java deleted file mode 100644 index d3bd28e84f..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java +++ /dev/null @@ -1,10 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; - -import java.util.Optional; - -public interface MemberRetriever { - - Optional findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(String collectionName, long sequenceNr); -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java index 0dc1bcb726..6453a49df3 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java @@ -1,17 +1,4 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; - -public record BucketRelation( - Bucket fromBucket, - Bucket toBucket, - String treeRelationType, - String treeValue, - String treeValueType, - String treePath -) { - public static BucketRelation createGenericRelation(Bucket fromBucket, Bucket toBucket) { - return new BucketRelation(fromBucket, toBucket, RdfConstants.GENERIC_TREE_RELATION, "", "", ""); - } -} +public record BucketRelation(String fromPartialUrl, String toPartialUrl, TreeRelation relation) { +} \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java deleted file mode 100644 index 27295eb786..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; - -public record BucketRelationCreatedEvent(BucketRelation bucketRelation) { -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java new file mode 100644 index 0000000000..c3efcde7f0 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java @@ -0,0 +1,11 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; + +public record TreeRelation(String treeRelationType, String treeValue, String treeValueType, String treePath) { + + public static TreeRelation generic() { + return new TreeRelation(RdfConstants.GENERIC_TREE_RELATION, "", "", ""); + } + +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java index e0c48359eb..a139eaa489 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java @@ -1,6 +1,5 @@ open module ldes.fragmentation.domain { requires ldes.ingest.domain; - requires ldes.server.retention; requires ldes.domain; requires spring.batch.core; requires spring.batch.infrastructure; @@ -16,7 +15,5 @@ requires spring.core; exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; - exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services; exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; - } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java index b9870856e7..35af7951dc 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java @@ -33,8 +33,8 @@ @RunWith(SpringRunner.class) @SpringBatchTest @EnableAutoConfiguration -@ContextConfiguration(classes = {SpringBatchConfiguration.class, FragmentationService.class }) -@TestPropertySource(properties = { "ldes-server.fragmentation-cron=*/1 * * * * *"}) +@ContextConfiguration(classes = {SpringBatchConfiguration.class, FragmentationService.class}) +@TestPropertySource(properties = {"ldes-server.fragmentation-cron=*/1 * * * * *"}) class FragmentationServiceTest { @MockBean(name = BUCKETISATION_STEP) Step bucketStep; @@ -87,8 +87,7 @@ void when_unprocessedViews_then_triggerJobsForEachViewThatIsntRunningAlready() t String collection = "collection"; when(memberMetricsRepository.getUnprocessedViews()) - .thenReturn(List.of(new ViewName(collection, "v1"), - new ViewName(collection, "v2"))); + .thenReturn(List.of(new ViewName(collection, "v1"), new ViewName(collection, "v2"))); JobExecution jobExecution = mock(JobExecution.class); JobParameters jobParameters = new JobParametersBuilder() diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java index 6c2aff5693..8100a0c520 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java @@ -7,7 +7,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyCreator; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import io.micrometer.observation.ObservationRegistry; import org.junit.jupiter.api.Test; @@ -21,10 +20,9 @@ class FragmentationStrategyBatchCollectionTest { private static final String COLLECTION_NAME = "collectionName"; private final FragmentationStrategyCreator fragmentationStrategyCreator = mock(FragmentationStrategyCreator.class); private final ObservationRegistry observationRegistry = mock(ObservationRegistry.class); - private final BucketRepository bucketRepository = mock(BucketRepository.class); private final FragmentationStrategyBatchCollection fragmentationStrategyCollection = new FragmentationStrategyBatchCollection( - bucketRepository, fragmentationStrategyCreator, observationRegistry); + fragmentationStrategyCreator, observationRegistry); @Test void when_ViewAddedEventIsReceived_FragmentationStrategyIsAddedToMap() { @@ -81,9 +79,8 @@ private InitViewAddedResult initAddView() { ViewSpecification viewSpecification = new ViewSpecification(viewName, List.of(), List.of(), 100); FragmentationStrategy fragmentationStrategy = mock(FragmentationStrategy.class); - final var rootBucketRetriever = new RootBucketRetriever(viewName, mock(), observationRegistry); FragmentationStrategyBatchExecutor fragmentationStrategyExecutor = - new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, rootBucketRetriever, observationRegistry); + new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, observationRegistry); return new InitViewAddedResult(viewName, viewSpecification, fragmentationStrategy, fragmentationStrategyExecutor); diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java index cebc5d811b..1f9824a5ac 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java @@ -30,8 +30,6 @@ class FragmentationStrategyBatchExecutorTest { private ExecutorService executorService; @Mock private FragmentationStrategy fragmentationStrategy; - @Mock - private RootBucketRetriever rootBucketRetriever; @Nested class ExecuteNext { @@ -41,16 +39,15 @@ class ExecuteNext { @Test void when_ExecuteIsCalled_then_AllLogicIsWrappedByTheExecutorService() { ObservationRegistry observationRegistry = ObservationRegistry.NOOP; - var executor = new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, rootBucketRetriever, + var executor = new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, observationRegistry); var member = mock(FragmentationMember.class); executor.bucketise(member); - verify(rootBucketRetriever).retrieveRootBucket(any()); verify(fragmentationStrategy).addMemberToBucket(any(), eq(member), any()); - verifyNoMoreInteractions(fragmentationStrategy, rootBucketRetriever); + verifyNoMoreInteractions(fragmentationStrategy); } } @@ -58,7 +55,7 @@ void when_ExecuteIsCalled_then_AllLogicIsWrappedByTheExecutorService() { void isPartOfCollection() { final ViewName viewNameA = ViewName.fromString("col/viewA"); var executorA = new FragmentationStrategyBatchExecutor(viewNameA, null, - null, null); + null); assertTrue(executorA.isPartOfCollection(viewNameA.getCollectionName())); assertFalse(executorA.isPartOfCollection("other")); @@ -68,7 +65,7 @@ void isPartOfCollection() { void getViewName() { final ViewName viewNameA = ViewName.fromString("col/viewA"); var executorA = new FragmentationStrategyBatchExecutor(viewNameA, null, - null, null); + null); assertEquals(viewNameA, executorA.getViewName()); } @@ -88,19 +85,19 @@ static class EqualityTestProvider implements ArgumentsProvider { private static final ViewName viewNameA = ViewName.fromString("col/viewA"); private static final FragmentationStrategyBatchExecutor executorA = new FragmentationStrategyBatchExecutor(viewNameA, - null, null, null); + null, null); @Override public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of(equals(), executorA, executorA), Arguments.of(equals(), executorA, - new FragmentationStrategyBatchExecutor(viewNameA, null, null, null)), + new FragmentationStrategyBatchExecutor(viewNameA, null, null)), Arguments.of(equals(), executorA, new FragmentationStrategyBatchExecutor(viewNameA, mock(FragmentationStrategy.class), - mock(RootBucketRetriever.class), mock(ObservationRegistry.class))), + mock(ObservationRegistry.class))), Arguments.of(notEquals(), executorA, - new FragmentationStrategyBatchExecutor(ViewName.fromString("col/viewB"), null, null, null))); + new FragmentationStrategyBatchExecutor(ViewName.fromString("col/viewB"), null, null))); } private static BiConsumer equals() { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java index fe1ce5a60b..37c9042f4a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java @@ -34,7 +34,7 @@ class FragmentationStrategyCreatorImplTest { @BeforeEach void setUp() { applicationContext = mock(ApplicationContext.class); - rootBucketCreator = mock(); + rootBucketCreator = mock(RootBucketCreator.class); fragmentationStrategyCreator = new FragmentationStrategyCreatorImpl(applicationContext, rootBucketCreator); } @@ -45,9 +45,9 @@ void when_ViewSpecificationFragmentationConfigIsNull_FragmentationStrategyImplIs FragmentationStrategy fragmentationStrategy = fragmentationStrategyCreator .createFragmentationStrategyForView(viewSpecification); - assertThat(fragmentationStrategy).isOfAnyClassIn(FragmentationStrategyImpl.class); + assertThat(fragmentationStrategy).isInstanceOf(FragmentationStrategyImpl.class); InOrder inOrder = inOrder(applicationContext, rootBucketCreator); - inOrder.verify(rootBucketCreator).createRootBucketForView(viewSpecification.getName()); + inOrder.verify(rootBucketCreator).createRootBucketForView(VIEW_NAME); inOrder.verifyNoMoreInteractions(); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java index 73cf04205c..f0fda9c0ab 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java @@ -4,16 +4,12 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; import io.micrometer.observation.Observation; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -23,20 +19,9 @@ class FragmentationStrategyDecoratorTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); @Mock private FragmentationStrategy fragmentationStrategy; - @Mock - private ApplicationEventPublisher applicationEventPublisher; @InjectMocks private FragmentationStrategyDecoratorTestImpl fragmentationStrategyDecorator; - @Test - void when_ParentDoesNotYetHaveRelationToChild_AddRelationAndSaveToDatabase() { - Bucket parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - Bucket childBucket = parentBucket.createChild(new BucketDescriptorPair("key", "value")); - - fragmentationStrategyDecorator.addRelationFromParentToChild(parentBucket, childBucket); - - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(BucketRelation.createGenericRelation(parentBucket, childBucket))); - } @Test void when_DecoratorAddsMemberToBucket_WrappedFragmentationStrategyIsCalled() { @@ -50,9 +35,8 @@ void when_DecoratorAddsMemberToBucket_WrappedFragmentationStrategyIsCalled() { } static class FragmentationStrategyDecoratorTestImpl extends FragmentationStrategyDecorator { - protected FragmentationStrategyDecoratorTestImpl(FragmentationStrategy fragmentationStrategy, - ApplicationEventPublisher applicationEventPublisher) { - super(fragmentationStrategy, applicationEventPublisher); + protected FragmentationStrategyDecoratorTestImpl(FragmentationStrategy fragmentationStrategy) { + super(fragmentationStrategy); } } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java index e86bd5de9d..13ae7e8f03 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java @@ -21,18 +21,14 @@ class FragmentationStrategyImplTest { private final FragmentationStrategyImpl fragmentationStrategy = new FragmentationStrategyImpl(); @Test - void when_memberIsAddedToBucket_FragmentationStrategyImplSavesUpdatedFragment() { - BucketisedMember expectedBucketisedMember = new BucketisedMember(BUCKET_ID, MEMBER_ID); - Bucket bucket = new Bucket(2, BucketDescriptor.empty(), VIEW_NAME); + void when_memberIsAddedToBucket_FragmentationStrategyImplAddsMemberToBucket() { + Bucket bucket = new Bucket(BUCKET_ID, BucketDescriptor.empty(), VIEW_NAME, List.of(), 0); FragmentationMember member = mock(FragmentationMember.class); + BucketisedMember expected = new BucketisedMember(BUCKET_ID, MEMBER_ID); when(member.getMemberId()).thenReturn(MEMBER_ID); - List members = fragmentationStrategy.addMemberToBucket(bucket, member, mock(Observation.class)); + fragmentationStrategy.addMemberToBucket(bucket, member, mock(Observation.class)); - assertThat(members) - .hasSize(1) - .first() - .usingRecursiveComparison() - .isEqualTo(expectedBucketisedMember); + assertThat(bucket.getMember()).contains(expected); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java deleted file mode 100644 index de488a0bcd..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingRootFragmentException; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RootBucketRetrieverTest { - - @Mock - private BucketRepository bucketRepository; - - @Mock - private Observation observation; - private RootBucketRetriever rootBucketRetriever; - private Bucket rootBucket; - private static final ViewName VIEW_NAME = new ViewName("collection", "view"); - - @BeforeEach - void setUp() { - rootBucketRetriever = new RootBucketRetriever(VIEW_NAME, bucketRepository, ObservationRegistry.NOOP); - rootBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - when(bucketRepository.retrieveRootBucket(VIEW_NAME)).thenReturn(Optional.of(rootBucket)); - } - - @Test - void should_FetchRootFragment_when_NotYetInMemory() { - final Bucket result = rootBucketRetriever.retrieveRootBucket(observation); - - assertEquals(rootBucket, result); - verify(bucketRepository).retrieveRootBucket(VIEW_NAME); - } - - @Test - void should_ReturnFragmentFromMemory_when_FetchedEarlier() { - rootBucketRetriever.retrieveRootBucket(observation); - rootBucketRetriever.retrieveRootBucket(observation); - rootBucketRetriever.retrieveRootBucket(observation); - final Bucket result = rootBucketRetriever.retrieveRootBucket(observation); - - assertEquals(rootBucket, result); - verify(bucketRepository).retrieveRootBucket(VIEW_NAME); - } - - @Test - void should_ThrowException_when_RootFragmentIsNotFound() { - when(bucketRepository.retrieveRootBucket(VIEW_NAME)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> rootBucketRetriever.retrieveRootBucket(observation)) - .isInstanceOf(MissingRootFragmentException.class) - .hasMessage("Could not retrieve root fragment for view %s", VIEW_NAME.asString()); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java index 661651e7a2..950be0136e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java @@ -3,6 +3,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.util.List; @@ -70,4 +73,51 @@ void testHashCode() { .isNotEqualTo(otherBucket.hashCode()); } + @Nested + class CompositionTest { + private static final int DAY_14_BUCKET_ID = 10; + private static final int DAY_28_BUCKET_ID = 11; + private Bucket rootBucket, year2023Bucket, year2024Bucket, month01Bucket, month03Bucket, day14Bucket, day28Bucket; + + @BeforeEach + void setUp() { + rootBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + year2023Bucket = rootBucket.createChild(new BucketDescriptorPair("year", "2023")); + year2024Bucket = rootBucket.createChild(new BucketDescriptorPair("year", "2024")); + month01Bucket = year2024Bucket.createChild(new BucketDescriptorPair("month", "01")); + month03Bucket = year2024Bucket.createChild(new BucketDescriptorPair("month", "03")); + day14Bucket = new Bucket(DAY_14_BUCKET_ID, month03Bucket.createChild(new BucketDescriptorPair("day", "14")).getBucketDescriptor(), VIEW_NAME, List.of(), 0); + day28Bucket = new Bucket(DAY_28_BUCKET_ID, month03Bucket.createChild(new BucketDescriptorPair("day", "28")).getBucketDescriptor(), VIEW_NAME, List.of(), 0); + month03Bucket.addChildBucket(day14Bucket.withGenericRelation()); + month03Bucket.addChildBucket(day28Bucket.withGenericRelation()); + year2024Bucket.addChildBucket(month01Bucket.withGenericRelation()); + year2024Bucket.addChildBucket(month03Bucket.withGenericRelation()); + rootBucket.addChildBucket(year2023Bucket.withGenericRelation()); + rootBucket.addChildBucket(year2024Bucket.withGenericRelation()); + } + + @Test + void test_GetBucketTree() { + final List descendants = rootBucket.getBucketTree(); + + assertThat(descendants).containsExactlyInAnyOrder( + rootBucket, year2023Bucket, year2024Bucket, month01Bucket, month03Bucket, day14Bucket, day28Bucket + ); + } + + @Test + void test_AddChild() { + rootBucket = Bucket.createRootBucketForView(VIEW_NAME); + rootBucket.addChildBucket(year2024Bucket.withGenericRelation()); + final Bucket duplicateBucket = rootBucket.createChild(new BucketDescriptorPair("year", "2024")); + TreeRelation treeRelation = new TreeRelation("duplicate-type", "duplicate-value", "duplicate-value-type", "duplicate-path"); + + rootBucket.addChildBucket(duplicateBucket.withRelations(treeRelation)); + + assertThat(rootBucket.getChildren()).hasSize(1); + assertThat(rootBucket.getChildRelations()) + .hasSize(2) + .satisfiesOnlyOnce(bucketRelation -> assertThat(bucketRelation.relation()).isEqualTo(treeRelation)); + } + } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java deleted file mode 100644 index 6ebd67a798..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class FragmentSequenceTest { - - @Test - void createNeverProcessedSequence() { - ViewName viewName = ViewName.fromString("col/view"); - var sequence = FragmentSequence.createNeverProcessedSequence(viewName); - - assertEquals(viewName, sequence.viewName()); - assertEquals(-1, sequence.sequenceNr()); - } -} \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java deleted file mode 100644 index e1885e7704..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java +++ /dev/null @@ -1,170 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.DuplicateFragmentPairException; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; - -import static be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier.fromFragmentId; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment.ROOT; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.junit.jupiter.api.Assertions.*; - -class FragmentTest { - private static final String VIEW = "mobility-hindrances"; - private static final ViewName VIEW_NAME = new ViewName("collectionName", VIEW); - private static final String FRAGMENTATION_VALUE_1 = "2020-12-28T09:36:09.72Z"; - private static final String GENERATED_AT_TIME = "generatedAtTime"; - private static final String FRAGMENTATION_VALUE_2 = "0/0/0"; - private static final String TILE = "tile"; - public static final FragmentPair PARENT_FRAGMENT_PAIR = new FragmentPair("a", "b"); - public static final FragmentPair CHILD_FRAGMENT_PAIR = new FragmentPair("c", "d"); - - @Test - void when_LdesFragmentIsImmutable_IsImmutableReturnsTrue() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier( - VIEW_NAME, - List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1)))); - assertFalse(fragment.isImmutable()); - fragment.makeImmutable(); - assertTrue(fragment.isImmutable()); - } - - @Test - void get_FragmentId() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier( - VIEW_NAME, - List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1), - new FragmentPair(TILE, FRAGMENTATION_VALUE_2)))); - - assertEquals("/collectionName/mobility-hindrances?generatedAtTime=2020-12-28T09:36:09.72Z&tile=0/0/0", - fragment.getFragmentIdString()); - assertEquals("/collectionName/mobility-hindrances?generatedAtTime=2020-12-28T09:36:09.72Z", - fragment.getParentIdAsString()); - - fragment = new Fragment(new LdesFragmentIdentifier( - VIEW_NAME, List.of())); - assertEquals("/collectionName/mobility-hindrances", - fragment.getFragmentIdString()); - assertEquals(ROOT, fragment.getParentIdAsString()); - - } - - @Test - void when_ValueIsAbsent_GetValueOfKeyReturnsOptionalEmpty() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, - List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1), - new FragmentPair(TILE, FRAGMENTATION_VALUE_2)))); - assertTrue(fragment.getValueOfKey("unexistingKey").isEmpty()); - assertEquals(Optional.of(FRAGMENTATION_VALUE_1), fragment.getValueOfKey(GENERATED_AT_TIME)); - assertEquals(Optional.of(FRAGMENTATION_VALUE_2), fragment.getValueOfKey(TILE)); - } - - @Test - void when_childIsCreated_ViewIsSameAndFragmentPairsAreExtended() { - ViewName viewName = VIEW_NAME; - Fragment fragment = new Fragment( - new LdesFragmentIdentifier(viewName, List.of(PARENT_FRAGMENT_PAIR))); - Fragment child = fragment.createChild(CHILD_FRAGMENT_PAIR); - assertEquals(List.of(PARENT_FRAGMENT_PAIR, CHILD_FRAGMENT_PAIR), child.getFragmentPairs()); - assertFalse(child.isImmutable()); - assertEquals(viewName, child.getViewName()); - } - - @Test - void when_LdesFragmentIsMadeImmutable_ImmutableTimeStampIsSet() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, - List.of(PARENT_FRAGMENT_PAIR))); - assertFalse(fragment.isImmutable()); - fragment.makeImmutable(); - assertTrue(fragment.isImmutable()); - } - - @Test - void when_ParentExists_Then_ReturnIdParent() { - Fragment parent = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of(PARENT_FRAGMENT_PAIR))); - Fragment child = parent.createChild(CHILD_FRAGMENT_PAIR); - assertThat(child.getParentId()).contains(parent.getFragmentId()); - assertThat(parent.getFragmentIdString()).isEqualTo(child.getParentIdAsString()); - } - - @Test - void when_FragmentPairExists_Then_ThrowError() { - Fragment parent = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of(PARENT_FRAGMENT_PAIR, CHILD_FRAGMENT_PAIR))); - assertThatExceptionOfType(DuplicateFragmentPairException.class) - .isThrownBy(() -> parent.createChild(CHILD_FRAGMENT_PAIR)) - .withMessage("FragmentId /collectionName/mobility-hindrances?a=b&c=d already contains fragmentkey c"); - } - - @Test - void when_ParentDoesNotExists_Then_ReturnEmpty() { - Fragment rootFragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of())); - assertEquals(Optional.empty(), rootFragment.getParentId()); - assertEquals(ROOT, rootFragment.getParentIdAsString()); - } - - @Test - void when_removeRelationToIdentifier_Then_ExpectItemToBeRemoved() { - Fragment a = new Fragment(fromFragmentId("/c/v")); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v1?k=1"))); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v1?k=2"))); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v1?k=3"))); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v2?k=1"))); - - LdesFragmentIdentifier toRemove = fromFragmentId("/c/v1?k=2"); - - assertEquals(4, a.getRelations().size()); - a.removeRelationToIdentifier(toRemove); - assertFalse(a.getRelations().stream().anyMatch(treeRelation -> treeRelation.treeNode().equals(toRemove))); - assertEquals(3, a.getRelations().size()); - } - - @Test - void testEquals() { - Fragment a = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment a2 = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment c = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "c"), List.of())); - - assertEquals(a, a2); - assertEquals(a2, a); - assertNotEquals(a, c); - } - - @Test - void testHashCode() { - Fragment a = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment a2 = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment c = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "c"), List.of())); - - assertEquals(a.hashCode(), a2.hashCode()); - assertEquals(a2.hashCode(), a.hashCode()); - assertNotEquals(a.hashCode(), c.hashCode()); - } - - @Test - void isConnectedTo() { - Fragment f3 = new Fragment(fromFragmentId("collection/3"), true, 0, List.of(), - null); - Fragment f2 = new Fragment(fromFragmentId("collection/1"), true, 0, List.of(), - null); - Fragment f1 = new Fragment(fromFragmentId("collection/2"), true, 0, - List.of(new TreeRelation(null, f2.getFragmentId(), null, null, null)), - null); - - assertFalse(f2.isConnectedTo(f1)); - assertTrue(f1.isConnectedTo(f2)); - assertFalse(f3.isConnectedTo(f2)); - assertFalse(f3.isConnectedTo(f1)); - } - - private TreeRelation createEmptyRelationForFragment(LdesFragmentIdentifier fragmentIdentifier) { - return new TreeRelation(null, fragmentIdentifier, null, null, null); - } - -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml index 9d3a1b8e3a..b09b62ffa1 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml @@ -3,7 +3,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.3.0 + 3.4.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java index 547717ded6..d579364c00 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java @@ -3,17 +3,11 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.bucketising.GeospatialBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.fragments.GeospatialBucketCreator; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.Collection; -import java.util.List; -import java.util.Set; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE_ROOT; @@ -24,54 +18,38 @@ public class GeospatialFragmentationStrategy extends FragmentationStrategyDecora private final GeospatialBucketCreator bucketCreator; private final ObservationRegistry observationRegistry; - private Bucket rootTileBucket = null; - public GeospatialFragmentationStrategy(FragmentationStrategy fragmentationStrategy, GeospatialBucketiser geospatialBucketiser, GeospatialBucketCreator bucketCreator, - ObservationRegistry observationRegistry, - ApplicationEventPublisher applicationEventPublisher) { - super(fragmentationStrategy, applicationEventPublisher); + ObservationRegistry observationRegistry) { + super(fragmentationStrategy); this.geospatialBucketiser = geospatialBucketiser; this.bucketCreator = bucketCreator; this.observationRegistry = observationRegistry; } @Override - public List addMemberToBucket(Bucket parentBucket, FragmentationMember member, - Observation parentObservation) { + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { Observation geospatialFragmentationObservation = Observation .createNotStarted("geospatial bucketisation", observationRegistry) .parentObservation(parentObservation) .start(); - setRootTileBucket(parentBucket); - - Set tiles = geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel()); - List buckets = tiles + geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel()) .stream() .map(tile -> { if (tile.equals(DEFAULT_BUCKET_STRING)) { - return bucketCreator.getOrCreateTileBucket(parentBucket, tile, parentBucket); + return bucketCreator.createTileBucket(parentBucket, tile, parentBucket); } else { - return bucketCreator.getOrCreateTileBucket(parentBucket, tile, rootTileBucket); + Bucket rootTileBucket = createRootTileBucket(parentBucket); + return bucketCreator.createTileBucket(parentBucket, tile, rootTileBucket); } - }).toList(); - - List members = buckets - .parallelStream() - .map(bucket -> super.addMemberToBucket(bucket, member, geospatialFragmentationObservation)) - .flatMap(Collection::stream) - .toList(); + }) + .forEach(bucket -> super.addMemberToBucket(bucket, member, geospatialFragmentationObservation)); geospatialFragmentationObservation.stop(); - return members; } - - private void setRootTileBucket(Bucket parentBucket) { - if (rootTileBucket == null) { - rootTileBucket = bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_TILE_ROOT); - super.addRelationFromParentToChild(parentBucket, rootTileBucket); - } + private Bucket createRootTileBucket(Bucket parentBucket) { + return parentBucket.addChildBucket(bucketCreator.createRootBucket(parentBucket, FRAGMENT_KEY_TILE_ROOT).withGenericRelation()); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java index 249527ba9b..574cf44af3 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.bucketising.GeospatialBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.config.GeospatialConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations.TileBucketRelationsAttributer; @@ -18,15 +17,14 @@ public class GeospatialFragmentationStrategyWrapper implements FragmentationStra public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, FragmentationStrategy fragmentationStrategy, ConfigProperties fragmentationProperties) { ObservationRegistry observationRegistry = applicationContext.getBean(ObservationRegistry.class); - BucketRepository bucketRepository = applicationContext.getBean(BucketRepository.class); - TileBucketRelationsAttributer tileBucketRelationsAttributer = new TileBucketRelationsAttributer(applicationContext); + TileBucketRelationsAttributer tileBucketRelationsAttributer = new TileBucketRelationsAttributer(); GeospatialConfig geospatialConfig = createGeospatialConfig(fragmentationProperties); GeospatialBucketiser geospatialBucketiser = new GeospatialBucketiser(geospatialConfig); - GeospatialBucketCreator geospatialBucketCreator = new GeospatialBucketCreator(bucketRepository, tileBucketRelationsAttributer); + GeospatialBucketCreator geospatialBucketCreator = new GeospatialBucketCreator(tileBucketRelationsAttributer); return new GeospatialFragmentationStrategy(fragmentationStrategy, - geospatialBucketiser, geospatialBucketCreator, observationRegistry, applicationContext); + geospatialBucketiser, geospatialBucketCreator, observationRegistry); } private GeospatialConfig createGeospatialConfig(ConfigProperties properties) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java index f92fffdbfa..e22e41e84d 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java @@ -26,11 +26,11 @@ public GeospatialBucketiser(GeospatialConfig geospatialConfig) { this.geospatialConfig = geospatialConfig; } - public Set bucketise(String memberId, Model memberModel) { + public Set createTiles(String memberId, Model memberModel) { try { Set tiles = getFragmentationObjects(memberModel, geospatialConfig.fragmenterSubjectFilter(), geospatialConfig.fragmentationPath()) .flatMap(geometryWrapper -> calculateTiles(geometryWrapper.getXYGeometry().toText(), geospatialConfig.maxZoom()).stream()) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); if(tiles.isEmpty()) { tiles.add(DEFAULT_BUCKET_STRING); diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java deleted file mode 100644 index 4c9d4b0271..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java +++ /dev/null @@ -1,53 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.BoundingBox; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.BoundingBoxConverter; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.TileConverter; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.model.Tile; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.*; - -public class GeospatialRelationsAttributer implements RelationsAttributer { - - public TreeRelation getRelationToParentFragment(Fragment childFragment) { - String targetWKT = getWKT(childFragment); - - return new TreeRelation(GEOSPARQL_AS_WKT, childFragment.getFragmentId(), - WGS_84 + " " + targetWKT, WKT_DATA_TYPE, TREE_GEOSPATIALLY_CONTAINS_RELATION); - } - - public BucketRelation createRelationBetween(Bucket parentBucket, Bucket childBucket) { - final String treeValue = WGS_84 + " " + getWKT(childBucket); - return new BucketRelation( - parentBucket, - childBucket, - TREE_GEOSPATIALLY_CONTAINS_RELATION, - treeValue, - WKT_DATA_TYPE, - GEOSPARQL_AS_WKT - ); - } - - private String getWKT(Fragment currentFragment) { - String fragmentWKT = currentFragment.getValueOfKey(FRAGMENT_KEY_TILE).orElseThrow( - () -> new MissingFragmentValueException(currentFragment.getFragmentIdString(), FRAGMENT_KEY_TILE)); - Tile currentTile = TileConverter.fromString(fragmentWKT); - BoundingBox currentBoundingBox = new BoundingBox(currentTile); - return BoundingBoxConverter.toWkt(currentBoundingBox); - } - - private String getWKT(Bucket currentBucket) { - String fragmentWKT = currentBucket.getValueForKey(FRAGMENT_KEY_TILE).orElseThrow( - () -> new MissingFragmentValueException(currentBucket.getBucketDescriptorAsString(), FRAGMENT_KEY_TILE)); - Tile currentTile = TileConverter.fromString(fragmentWKT); - BoundingBox currentBoundingBox = new BoundingBox(currentTile); - return BoundingBoxConverter.toWkt(currentBoundingBox); - } - -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java index 4f5a9d754e..2706ab63f1 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java @@ -1,25 +1,40 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; -import org.springframework.context.ApplicationEventPublisher; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.BoundingBox; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.BoundingBoxConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.TileConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.model.Tile; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; +import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.*; -public class TileBucketRelationsAttributer { +public class TileBucketRelationsAttributer implements RelationsAttributer { - private final GeospatialRelationsAttributer relationsAttributer = new GeospatialRelationsAttributer(); - private final ApplicationEventPublisher applicationEventPublisher; + public Bucket addRelationsFromRootToBottom(Bucket rootBucket, Bucket tileBucket) { + boolean isDefaultBucket = tileBucket.getValueForKey(FRAGMENT_KEY_TILE).orElse("").equals(DEFAULT_BUCKET_STRING); + TreeRelation treeRelation = isDefaultBucket ? TreeRelation.generic() : createGeospatialRelationToParent(tileBucket); + return rootBucket.addChildBucket(tileBucket.withRelations(treeRelation)); + } - public TileBucketRelationsAttributer(ApplicationEventPublisher applicationEventPublisher) { - this.applicationEventPublisher = applicationEventPublisher; + private TreeRelation createGeospatialRelationToParent(Bucket childBucket) { + final String treeValue = WGS_84 + " " + getWKT(childBucket); + return new TreeRelation( + TREE_GEOSPATIALLY_CONTAINS_RELATION, + treeValue, + WKT_DATA_TYPE, + GEOSPARQL_AS_WKT + ); } - public void addRelationsFromRootToBottom(Bucket rootBucket, Bucket tileBucket) { - boolean isDefaultBucket = tileBucket.getValueForKey(FRAGMENT_KEY_TILE).orElse("").equals(DEFAULT_BUCKET_STRING); - BucketRelation bucketRelation = isDefaultBucket ? BucketRelation.createGenericRelation(rootBucket, tileBucket) : relationsAttributer.createRelationBetween(rootBucket, tileBucket); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + private String getWKT(Bucket currentBucket) { + String fragmentWKT = currentBucket.getValueForKey(FRAGMENT_KEY_TILE).orElseThrow( + () -> new MissingFragmentValueException(currentBucket.getBucketDescriptorAsString(), FRAGMENT_KEY_TILE)); + Tile currentTile = TileConverter.fromString(fragmentWKT); + BoundingBox currentBoundingBox = new BoundingBox(currentTile); + return BoundingBoxConverter.toWkt(currentBoundingBox); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java deleted file mode 100644 index ae293f31b2..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java +++ /dev/null @@ -1,19 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.exceptions; - -import org.apache.jena.geosparql.implementation.GeometryWrapper; - -public class RdfGeometryException extends RuntimeException { - private final GeometryWrapper geometryWrapper; - private final String srsFormat; - - public RdfGeometryException(GeometryWrapper geometryWrapper, String srsFormat, Exception e) { - super(e); - this.geometryWrapper = geometryWrapper; - this.srsFormat = srsFormat; - } - - @Override - public String getMessage() { - return "unable to convert member" + geometryWrapper + " to " + srsFormat + "format"; - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java index c0590a62f1..1abd977877 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java @@ -1,45 +1,27 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.fragments; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations.TileBucketRelationsAttributer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; public class GeospatialBucketCreator { - private final BucketRepository bucketRepository; private final TileBucketRelationsAttributer tileBucketRelationsAttributer; - private static final Logger LOGGER = LoggerFactory.getLogger(GeospatialBucketCreator.class); - public GeospatialBucketCreator(BucketRepository bucketRepository, TileBucketRelationsAttributer tileBucketRelationsAttributer) { - this.bucketRepository = bucketRepository; + public GeospatialBucketCreator(TileBucketRelationsAttributer tileBucketRelationsAttributer) { this.tileBucketRelationsAttributer = tileBucketRelationsAttributer; } - public Bucket getOrCreateTileBucket(Bucket parentBucket, String tile, Bucket rootTileFragment) { + public Bucket createTileBucket(Bucket parentBucket, String tile, Bucket rootTileFragment) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile); - return bucketRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = bucketRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileFragment, childBucket); - LOGGER.debug("Geospatial fragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + final Bucket childBucket = parentBucket.createChild(childDescriptorPair); + return tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileFragment, childBucket); } - public Bucket getOrCreateRootBucket(Bucket parentBucket, String tile) { + public Bucket createRootBucket(Bucket parentBucket, String tile) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile); - return bucketRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = bucketRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - LOGGER.debug("Geospatial rootfragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + return parentBucket.createChild(childDescriptorPair); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java index 2e9436c09a..56a95fc3f1 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java @@ -18,14 +18,15 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE_ROOT; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; class GeospatialFragmentationStrategyTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - private static final Bucket ROOT_TILE_BUCKET = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT)); + private Bucket parentBucket; + private Bucket rootTileBucket; private GeospatialBucketiser geospatialBucketiser; private GeospatialBucketCreator bucketCreator; private FragmentationStrategy decoratedFragmentationStrategy; @@ -33,29 +34,46 @@ class GeospatialFragmentationStrategyTest { @BeforeEach void setUp() { + parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + rootTileBucket = parentBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT)); + geospatialBucketiser = mock(GeospatialBucketiser.class); bucketCreator = mock(GeospatialBucketCreator.class); decoratedFragmentationStrategy = mock(FragmentationStrategy.class); - when(bucketCreator.getOrCreateRootBucket(PARENT_BUCKET, FRAGMENT_KEY_TILE_ROOT)).thenReturn(ROOT_TILE_BUCKET); + when(bucketCreator.createRootBucket(parentBucket, FRAGMENT_KEY_TILE_ROOT)).thenReturn(rootTileBucket); geospatialFragmentationStrategy = new GeospatialFragmentationStrategy(decoratedFragmentationStrategy, - geospatialBucketiser, bucketCreator, ObservationRegistry.create(), - mock()); + geospatialBucketiser, bucketCreator, ObservationRegistry.create()); } - + + @Test + void when_RootTileBucketIsNotSet_then_SetRootTileBucket() { + FragmentationMember member = mock(FragmentationMember.class); + + when(geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel())).thenReturn(Set.of("dummy")); + when(bucketCreator.createTileBucket(parentBucket, "dummy", parentBucket)) + .thenReturn(mock(Bucket.class)); + + geospatialFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); + + verify(decoratedFragmentationStrategy).addMemberToBucket(any(), any(), any(Observation.class)); + verifyNoMoreInteractions(decoratedFragmentationStrategy); + assertThat(parentBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(rootTileBucket.withGenericRelation()); + } + @Test void when_MemberIsAddedToDefaultFragment_GeospatialFragmentationIsApplied() { FragmentationMember member = mock(FragmentationMember.class); - when(geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel())).thenReturn(Set.of(DEFAULT_BUCKET_STRING)); - Bucket defaultTileBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING)); - when(bucketCreator.getOrCreateTileBucket(PARENT_BUCKET, DEFAULT_BUCKET_STRING, PARENT_BUCKET)) + when(geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel())).thenReturn(Set.of(DEFAULT_BUCKET_STRING)); + Bucket defaultTileBucket = parentBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING)); + when(bucketCreator.createTileBucket(parentBucket, DEFAULT_BUCKET_STRING, parentBucket)) .thenReturn(defaultTileBucket); - geospatialFragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); + geospatialFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(defaultTileBucket), - any(), any(Observation.class)); + verify(decoratedFragmentationStrategy).addMemberToBucket(eq(defaultTileBucket), any(), any(Observation.class)); verifyNoMoreInteractions(decoratedFragmentationStrategy); } @@ -63,13 +81,13 @@ void when_MemberIsAddedToDefaultFragment_GeospatialFragmentationIsApplied() { void when_MemberIsAddedToBucket_GeospatialFragmentationIsApplied() { FragmentationMember member = mock(FragmentationMember.class); - when(geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel())).thenReturn(Set.of("1/1/1", + when(geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel())).thenReturn(Set.of("1/1/1", "2/2/2", "3/3/3")); Bucket tileBucketOne = mockCreationGeospatialBucket("1/1/1"); Bucket tileBucketTwo = mockCreationGeospatialBucket("2/2/2"); Bucket tileBucketThree = mockCreationGeospatialBucket("3/3/3"); - geospatialFragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); + geospatialFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(tileBucketOne), any(), any(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(tileBucketTwo), any(), any(Observation.class)); @@ -77,24 +95,9 @@ void when_MemberIsAddedToBucket_GeospatialFragmentationIsApplied() { verifyNoMoreInteractions(decoratedFragmentationStrategy); } - @Test - void when_MemberIsAddedToDefaultBucket_GeospatialFragmentationIsApplied() { - FragmentationMember member = mock(FragmentationMember.class); - - when(geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel())).thenReturn(Set.of(DEFAULT_BUCKET_STRING)); - Bucket defaultTileBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING)); - when(bucketCreator.getOrCreateTileBucket(PARENT_BUCKET, DEFAULT_BUCKET_STRING, PARENT_BUCKET)) - .thenReturn(defaultTileBucket); - - geospatialFragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); - - verify(decoratedFragmentationStrategy).addMemberToBucket(eq(defaultTileBucket), any(), any(Observation.class)); - verifyNoMoreInteractions(decoratedFragmentationStrategy); - } - private Bucket mockCreationGeospatialBucket(String tile) { - Bucket tileBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile)); - when(bucketCreator.getOrCreateTileBucket(PARENT_BUCKET, tile, ROOT_TILE_BUCKET)).thenReturn(tileBucket); + Bucket tileBucket = parentBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile)); + when(bucketCreator.createTileBucket(parentBucket, tile, rootTileBucket)).thenReturn(tileBucket); return tileBucket; } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java index bf1dc9986d..94407154bb 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java @@ -8,7 +8,7 @@ import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; class GeospatialFragmentationStrategyWrapperTest { @@ -26,8 +26,10 @@ void setUp() { void when_FragmentationStrategyIsUpdated_GeospatialFragmentationStrategyIsReturned() { ConfigProperties properties = new ConfigProperties( Map.of("maxZoom", "15", "fragmentationPath", "http://www.opengis.net/ont/geosparql#asWKT")); + FragmentationStrategy decoratedFragmentationStrategy = geospatialFragmentationUpdater .wrapFragmentationStrategy(applicationContext, fragmentationStrategy, properties); - assertTrue(decoratedFragmentationStrategy instanceof GeospatialFragmentationStrategy); + + assertThat(decoratedFragmentationStrategy).isInstanceOf(GeospatialFragmentationStrategy.class); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java index ddac4cc0e0..349dfbea22 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java @@ -40,7 +40,7 @@ void when_MemberIsBucketized_CorrectBucketsAreReturned() throws URISyntaxExcepti FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-bucketising.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(4, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -56,7 +56,7 @@ void when_MemberWith2GeoPropertiesIsBucketized_CorrectBucketsAreReturned() FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-2-geo-props-bucketising.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(2, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -72,7 +72,7 @@ void when_MemberWithIncorrectGeospatialProperty_Then_DefaultBucketIsReturned() FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-bucketising-faulty.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(1, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -88,7 +88,7 @@ void when_MemberWith1FaultyGeoPropertyIsBucketized_CorrectBucketsAreReturned() FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-2-geo-props-bucketising-faulty.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(1, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -103,7 +103,7 @@ void when_MemberHasInvalidCoordinates_DefaultBucketIsReturned() throws URISyntax FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-bucketising-invalid-coordinates.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(1, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java deleted file mode 100644 index a74d7a1e87..0000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class GeospatialRelationsAttributerTest { - - private final GeospatialRelationsAttributer geospatialRelationsAttributer = new GeospatialRelationsAttributer(); - private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Fragment CHILD_FRAGMENT = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, - List.of(new FragmentPair(FRAGMENT_KEY_TILE, - "1/1/1")))); - - private static final TreeRelation EXPECTED_RELATION = new TreeRelation("http://www.opengis.net/ont/geosparql#asWKT", - LdesFragmentIdentifier.fromFragmentId("/collectionName/view?tile=1/1/1"), - " POLYGON ((180 0, 180 -85.0511287798066, 0 -85.0511287798066, 0 0, 180 0))", - "http://www.opengis.net/ont/geosparql#wktLiteral", - "https://w3id.org/tree#GeospatiallyContainsRelation"); - - @Test - void when_getRelation_GeospatialRelationIsReturned() { - TreeRelation relationToParentFragment = geospatialRelationsAttributer.getRelationToParentFragment( - CHILD_FRAGMENT); - assertEquals(EXPECTED_RELATION, relationToParentFragment); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java index a9b687f847..2f5b014064 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java @@ -4,37 +4,34 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; -import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; @ExtendWith(MockitoExtension.class) class TileBucketRelationsAttributerTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - @Mock - private ApplicationEventPublisher applicationEventPublisher; - @InjectMocks + private TileBucketRelationsAttributer tileBucketRelationsAttributer; + @BeforeEach + void setUp() { + tileBucketRelationsAttributer = new TileBucketRelationsAttributer(); + } @Test void when_TileFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdded() { Bucket rootBucket = createTileBucket("0/0/0"); Bucket tileBucket = createTileBucket("1/1/1"); - BucketRelation bucketRelation = new BucketRelation( - rootBucket, - tileBucket, + TreeRelation treeRelation = new TreeRelation( "https://w3id.org/tree#GeospatiallyContainsRelation", " POLYGON ((180 0, 180 -85.0511287798066, 0 -85.0511287798066, 0 0, 180 0))", "http://www.opengis.net/ont/geosparql#wktLiteral", @@ -43,19 +40,21 @@ void when_TileFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdde tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootBucket, tileBucket); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(tileBucket.withRelations(treeRelation)); } @Test void when_DefaultFragmentIsCreated_RelationsBetweenRootAndCreatedFragmentIsAdded() { Bucket rootBucket = createTileBucket("0/0/0"); Bucket tileBucket = createTileBucket(DEFAULT_BUCKET_STRING); - BucketRelation bucketRelation = BucketRelation.createGenericRelation(rootBucket, tileBucket); tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootBucket, tileBucket); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(bucketRelation)); - + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(tileBucket.withGenericRelation()); } private Bucket createTileBucket(String tile) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java index bc99da427c..83d242e41f 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java @@ -2,16 +2,16 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations.TileBucketRelationsAttributer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Optional; - import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE_ROOT; @@ -22,110 +22,57 @@ class GeospatialBucketCreatorTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final BucketDescriptorPair timebasedPair = new BucketDescriptorPair("year", "2023"); - private static final BucketDescriptorPair geoRootPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT); - private static final BucketDescriptorPair geoPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, "15/101/202"); - private static final BucketDescriptorPair defaultPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING); + private BucketDescriptorPair geoRootPair; + private BucketDescriptorPair geoPair; + private BucketDescriptorPair defaultPair; - private BucketRepository bucketRepository; + @Mock + private TileBucketRelationsAttributer tileBucketRelationsAttributer; + @InjectMocks private GeospatialBucketCreator geospatialBucketCreator; @BeforeEach void setUp() { - bucketRepository = mock(BucketRepository.class); - geospatialBucketCreator = new GeospatialBucketCreator(bucketRepository, mock()); + geoRootPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT); + geoPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, "15/101/202"); + defaultPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING); } @Test - void when_TileFragmentDoesNotExist_NewTileFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket childBucket = bucket.createChild(geoPair); - - when(bucketRepository.retrieveBucket(VIEW_NAME, childBucket.getBucketDescriptor())) - .thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(childBucket); + void test_createTileBucket() { + Bucket rootBucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); + Bucket rootTileBucket = rootBucket.createChild(geoRootPair); + Bucket tileBucket = rootBucket.createChild(geoPair); + String tile = "15/101/202"; + when(tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileBucket, tileBucket)) + .thenReturn(rootBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile))); - Bucket returnedBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, "15/101/202", rootBucket); - - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(childBucket); - verify(bucketRepository).retrieveBucket(VIEW_NAME, childBucket.getBucketDescriptor()); - } - - @Test - void when_TileFragmentDoesNotExist_RetrievedTileFragmentIsReturned() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket tileBucket = bucket.createChild(geoPair); - - when(bucketRepository.retrieveBucket(VIEW_NAME, tileBucket.getBucketDescriptor())) - .thenReturn(Optional.of(tileBucket)); - - Bucket childBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, "15/101/202", rootBucket); + Bucket childBucket = geospatialBucketCreator.createTileBucket(rootBucket, "15/101/202", rootTileBucket); assertThat(childBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&tile=15/101/202"); - verify(bucketRepository).retrieveBucket(VIEW_NAME, tileBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(tileBucketRelationsAttributer).addRelationsFromRootToBottom(rootTileBucket, tileBucket); } @Test - void when_RootFragmentDoesNotExist_NewRootFragmentIsCreatedAndSaved() { + void test_CreateRootBucket() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(rootBucket); - - Bucket returnedBucket = geospatialBucketCreator.getOrCreateRootBucket(bucket, FRAGMENT_KEY_TILE_ROOT); - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(rootBucket); - verify(bucketRepository).retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor()); - } - - @Test - void when_RootFragmentDoesNotExist_RetrievedRootFragmentIsReturned() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor())) - .thenReturn(Optional.of(rootBucket)); + Bucket returnedBucket = geospatialBucketCreator.createRootBucket(bucket, FRAGMENT_KEY_TILE_ROOT); - Bucket returnedBucket = geospatialBucketCreator.getOrCreateRootBucket(bucket, FRAGMENT_KEY_TILE_ROOT); assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&tile=0/0/0"); - verify(bucketRepository).retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verifyNoInteractions(tileBucketRelationsAttributer); } @Test - void when_DefaultFragmentDoesNotExist_NewDefaultFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket defaultBucket = bucket.createChild(defaultPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(defaultBucket); - - Bucket returnedBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); - - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(defaultBucket); - assertThat(returnedBucket.getBucketDescriptor()).isEqualTo(defaultBucket.getBucketDescriptor()); - verify(bucketRepository).retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor()); - } - - @Test - void when_DefaultFragmentDoesNotExist_RetrievedDefaultFragmentIsReturned() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket defaultBucket = bucket.createChild(defaultPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor())) - .thenReturn(Optional.of(defaultBucket)); + void test_GetOrCreateDefaultTileBucket() { + Bucket rootbucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); + Bucket rootTileBucket = rootbucket.createChild(geoRootPair); + Bucket defaultBucket = rootbucket.createChild(defaultPair); + when(tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileBucket, defaultBucket)).thenReturn(defaultBucket); - Bucket returnedBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); + Bucket returnedBucket = geospatialBucketCreator.createTileBucket(rootbucket, DEFAULT_BUCKET_STRING, rootTileBucket); assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&tile=unknown"); - verify(bucketRepository).retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(tileBucketRelationsAttributer).addRelationsFromRootToBottom(rootTileBucket, defaultBucket); } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml index a408bc2d10..1cb9db9656 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.3.0 + 3.4.0-SNAPSHOT 4.0.0 jar diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java index 10ec7ada6b..ecc504aa62 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java @@ -3,68 +3,55 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.bucketising.ReferenceBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.Collection; -import java.util.List; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator.FRAGMENT_KEY_REFERENCE_ROOT; public class ReferenceFragmentationStrategy extends FragmentationStrategyDecorator { - public static final String REFERENCE_FRAGMENTATION = "ReferenceFragmentation"; - - private final ReferenceBucketiser referenceBucketiser; - private final ReferenceBucketCreator bucketCreator; - private final ObservationRegistry observationRegistry; - - public ReferenceFragmentationStrategy(FragmentationStrategy fragmentationStrategy, - ReferenceBucketiser referenceBucketiser, - ReferenceBucketCreator bucketCreator, - ObservationRegistry observationRegistry, - ApplicationEventPublisher applicationEventPublisher) { - super(fragmentationStrategy, applicationEventPublisher); - this.referenceBucketiser = referenceBucketiser; - this.bucketCreator = bucketCreator; - this.observationRegistry = observationRegistry; - } - - @Override - public List addMemberToBucket(Bucket parentBucket, FragmentationMember member, - Observation parentObservation) { - final var fragmentationObservation = startObservation(parentObservation); - final var rootFragment = getOrCreateRootBucket(parentBucket); - var fragments = - referenceBucketiser - .bucketise(member.getSubject(), member.getVersionModel()) - .stream() - .map(reference -> bucketCreator.getOrCreateBucket(parentBucket, reference, rootFragment)) - .toList(); - - List members = fragments.parallelStream() - .map(bucket -> super.addMemberToBucket(bucket, member, fragmentationObservation)) - .flatMap(Collection::stream) - .toList(); - fragmentationObservation.stop(); - return members; - } - - private Observation startObservation(Observation parentObservation) { - return Observation.createNotStarted("reference fragmentation", observationRegistry) - .parentObservation(parentObservation) - .start(); - } - - private Bucket getOrCreateRootBucket(Bucket parentBucket) { - Bucket referenceRootFragment = bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_REFERENCE_ROOT); - super.addRelationFromParentToChild(parentBucket, referenceRootFragment); - return referenceRootFragment; - } + public static final String REFERENCE_FRAGMENTATION = "ReferenceFragmentation"; + + private final ReferenceBucketiser referenceBucketiser; + private final ReferenceBucketCreator bucketCreator; + private final ObservationRegistry observationRegistry; + + public ReferenceFragmentationStrategy(FragmentationStrategy fragmentationStrategy, + ReferenceBucketiser referenceBucketiser, + ReferenceBucketCreator bucketCreator, + ObservationRegistry observationRegistry) { + super(fragmentationStrategy); + this.referenceBucketiser = referenceBucketiser; + this.bucketCreator = bucketCreator; + this.observationRegistry = observationRegistry; + } + + @Override + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { + final var fragmentationObservation = startObservation(parentObservation); + final var rootBucket = getOrCreateRootBucket(parentBucket); + + referenceBucketiser + .createReferences(member.getSubject(), member.getVersionModel()) + .stream() + .map(reference -> bucketCreator.getOrCreateBucket(parentBucket, reference, rootBucket)) + .forEach(bucket -> super.addMemberToBucket(bucket, member, fragmentationObservation)); + + fragmentationObservation.stop(); + } + + private Observation startObservation(Observation parentObservation) { + return Observation.createNotStarted("reference fragmentation", observationRegistry) + .parentObservation(parentObservation) + .start(); + } + + private Bucket getOrCreateRootBucket(Bucket parentBucket) { + Bucket referenceRootFragment = bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_REFERENCE_ROOT); + return parentBucket.addChildBucket(referenceRootFragment.withGenericRelation()); + } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java index 523ecf66ff..6bbae30efe 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.bucketising.ReferenceBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.config.ReferenceConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator; @@ -23,20 +22,19 @@ public class ReferenceFragmentationStrategyWrapper implements FragmentationStrat public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, FragmentationStrategy fragmentationStrategy, ConfigProperties properties) { final var fragmentationPath = properties.getOrDefault(FRAGMENTATION_PATH, DEFAULT_FRAGMENTATION_PATH); - final var bucketRepository = applicationContext.getBean(BucketRepository.class); final var observationRegistry = applicationContext.getBean(ObservationRegistry.class); final var referenceConfig = new ReferenceConfig(fragmentationPath); final var referenceBucketiser = new ReferenceBucketiser(referenceConfig); final var fragmentationKey = properties.getOrDefault(FRAGMENTATION_KEY, DEFAULT_FRAGMENTATION_KEY); - final var relationsAttributer = new ReferenceFragmentRelationsAttributer(applicationContext, fragmentationPath, fragmentationKey); + final var relationsAttributer = new ReferenceFragmentRelationsAttributer(fragmentationPath, fragmentationKey); - final var referenceBucketCreator = new ReferenceBucketCreator(bucketRepository, relationsAttributer, fragmentationKey); + final var referenceBucketCreator = new ReferenceBucketCreator(relationsAttributer, fragmentationKey); return new ReferenceFragmentationStrategy( fragmentationStrategy, referenceBucketiser, referenceBucketCreator, - observationRegistry, - applicationContext); + observationRegistry + ); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java index f6aa98b997..6e0843c4af 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java @@ -25,7 +25,7 @@ public ReferenceBucketiser(ReferenceConfig referenceConfig) { this.fragmentationPath = referenceConfig.fragmentationPath(); } - public Set bucketise(@NotNull String subject, @NotNull Model memberModel) { + public Set createReferences(@NotNull String subject, @NotNull Model memberModel) { try { Set references = memberModel .listObjectsOfProperty(createResource(subject), createProperty(fragmentationPath)) diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java index 0d94e3243b..e2d3da8321 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java @@ -1,55 +1,35 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.relations.ReferenceFragmentRelationsAttributer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; public class ReferenceBucketCreator { - private final String fragmentKeyReference; public static final String FRAGMENT_KEY_REFERENCE_ROOT = ""; - private static final Logger LOGGER = LoggerFactory.getLogger(ReferenceBucketCreator.class); - private final BucketRepository fragmentRepository; private final ReferenceFragmentRelationsAttributer relationsAttributer; - public ReferenceBucketCreator(BucketRepository fragmentRepository, - ReferenceFragmentRelationsAttributer relationsAttributer, + public ReferenceBucketCreator(ReferenceFragmentRelationsAttributer relationsAttributer, String fragmentKeyReference) { - this.fragmentRepository = fragmentRepository; this.relationsAttributer = relationsAttributer; this.fragmentKeyReference = fragmentKeyReference; } public Bucket getOrCreateBucket(Bucket parentBucket, String reference, Bucket rootBucket) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(fragmentKeyReference, reference); - return fragmentRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = fragmentRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - if (reference.equals(DEFAULT_BUCKET_STRING)) { - relationsAttributer.addDefaultRelation(parentBucket, childBucket); - } else { - relationsAttributer.addRelationFromRootToBottom(rootBucket, childBucket); - } - LOGGER.debug("Reference fragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + final Bucket childBucket = parentBucket.createChild(childDescriptorPair); + if (reference.equals(DEFAULT_BUCKET_STRING)) { + return relationsAttributer.addDefaultRelation(parentBucket, childBucket); + } else { + return relationsAttributer.addRelationFromRootToBottom(rootBucket, childBucket); + } } public Bucket getOrCreateRootBucket(Bucket parentBucket, String reference) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(fragmentKeyReference, reference); - return fragmentRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = fragmentRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - LOGGER.debug("Reference fragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + return parentBucket.createChild(childDescriptorPair); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java index 8fe48e310b..4301a4b659 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java @@ -3,10 +3,8 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import org.apache.jena.datatypes.xsd.XSDDatatype; -import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.TREE; @@ -14,34 +12,27 @@ public class ReferenceFragmentRelationsAttributer implements RelationsAttributer public static final String TREE_REFERENCE_EQUALS_RELATION = TREE + "EqualToRelation"; - private final ApplicationEventPublisher applicationEventPublisher; private final String fragmentationPath; private final String fragmentKeyReference; - public ReferenceFragmentRelationsAttributer(ApplicationEventPublisher applicationEventPublisher, - String fragmentationPath, - String fragmentKeyReference) { - this.applicationEventPublisher = applicationEventPublisher; + public ReferenceFragmentRelationsAttributer(String fragmentationPath, String fragmentKeyReference) { this.fragmentationPath = fragmentationPath; this.fragmentKeyReference = fragmentKeyReference; } - public void addRelationFromRootToBottom(Bucket rootBucket, Bucket referenceBucket) { - final BucketRelation bucketRelation = new BucketRelation( - rootBucket, - referenceBucket, + public Bucket addRelationFromRootToBottom(Bucket rootBucket, Bucket referenceBucket) { + final TreeRelation treeRelation = new TreeRelation( TREE_REFERENCE_EQUALS_RELATION, getTreeValue(referenceBucket), XSDDatatype.XSDanyURI.getURI(), fragmentationPath ); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + return rootBucket.addChildBucket(referenceBucket.withRelations(treeRelation)); } - public void addDefaultRelation(Bucket rootBucket, Bucket referenceBucket) { - final BucketRelation defaultRelation = BucketRelation.createGenericRelation(rootBucket, referenceBucket); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(defaultRelation)); + public Bucket addDefaultRelation(Bucket rootBucket, Bucket referenceBucket) { + return rootBucket.addChildBucket(referenceBucket.withGenericRelation()); } private String getTreeValue(Bucket currentBucket) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java index cfa6b07c0c..af59ef9d73 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java @@ -24,8 +24,11 @@ class ReferenceFragmentationStrategyTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - private static final Bucket ROOT_TILE_BUCKET = PARENT_BUCKET.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, FRAGMENT_KEY_REFERENCE_ROOT)); + private static final String TYPE_PARCEL = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; + private static final String TYPE_BUILDING = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"; + private static final String TYPE_ADDRESS = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Adres"; + private Bucket parentBucket; + private Bucket rootTileBucket; private ReferenceBucketiser referenceBucketiser; private ReferenceBucketCreator bucketCreator; @@ -34,60 +37,28 @@ class ReferenceFragmentationStrategyTest { @BeforeEach void setUp() { + parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + rootTileBucket = parentBucket.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, FRAGMENT_KEY_REFERENCE_ROOT)); + referenceBucketiser = mock(ReferenceBucketiser.class); bucketCreator = mock(ReferenceBucketCreator.class); decoratedFragmentationStrategy = mock(FragmentationStrategy.class); - when(bucketCreator.getOrCreateRootBucket(PARENT_BUCKET, FRAGMENT_KEY_REFERENCE_ROOT)) - .thenReturn(ROOT_TILE_BUCKET); + when(bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_REFERENCE_ROOT)) + .thenReturn(rootTileBucket); referenceFragmentationStrategy = new ReferenceFragmentationStrategy(decoratedFragmentationStrategy, - referenceBucketiser, bucketCreator, ObservationRegistry.create(), mock()); - } - - @Test - void when_MemberIsAddedToFragment_ThenReferenceFragmentationIsApplied() { - FragmentationMember member = mock(FragmentationMember.class); - - final var typePerceel = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; - final var typeGebouw = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"; - final var typeAdres = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Adres"; - - when(referenceBucketiser.bucketise(member.getSubject(), member.getVersionModel())) - .thenReturn(Set.of(typePerceel, typeGebouw, typeAdres)); - Bucket referenceBucketOne = mockCreationReferenceBucket(typePerceel); - Bucket referenceBucketTwo = mockCreationReferenceBucket(typeGebouw); - Bucket referenceBucketThree = mockCreationReferenceBucket(typeAdres); - - referenceFragmentationStrategy - .addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); - - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(referenceBucketOne), - any(), any(Observation.class)); - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(referenceBucketTwo), - any(), any(Observation.class)); - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(referenceBucketThree), - any(), any(Observation.class)); - verifyNoMoreInteractions(decoratedFragmentationStrategy); + referenceBucketiser, bucketCreator, ObservationRegistry.create()); } @Test void when_MemberIsAddedToBucket_ThenReferenceFragmentationIsApplied() { FragmentationMember member = mock(FragmentationMember.class); + when(referenceBucketiser.createReferences(member.getSubject(), member.getVersionModel())) + .thenReturn(Set.of(TYPE_PARCEL, TYPE_BUILDING, TYPE_ADDRESS)); + Bucket referenceBucketOne = mockCreationReferenceBucket(TYPE_PARCEL); + Bucket referenceBucketTwo = mockCreationReferenceBucket(TYPE_BUILDING); + Bucket referenceBucketThree = mockCreationReferenceBucket(TYPE_ADDRESS); - final var typeParcel = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; - final var typeBuilding = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"; - final var typeAddress = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Adres"; - - when(referenceBucketiser.bucketise(member.getSubject(), member.getVersionModel())) - .thenReturn(Set.of(typeParcel, typeBuilding, typeAddress)); - Bucket referenceBucketOne = mockCreationReferenceBucket(typeParcel); - Bucket referenceBucketTwo = mockCreationReferenceBucket(typeBuilding); - Bucket referenceBucketThree = mockCreationReferenceBucket(typeAddress); - - referenceFragmentationStrategy - .addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); + referenceFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(referenceBucketOne), any(), any(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(referenceBucketTwo), any(), any(Observation.class)); @@ -96,8 +67,8 @@ void when_MemberIsAddedToBucket_ThenReferenceFragmentationIsApplied() { } private Bucket mockCreationReferenceBucket(String tile) { - Bucket referenceBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, tile)); - when(bucketCreator.getOrCreateBucket(PARENT_BUCKET, tile, ROOT_TILE_BUCKET)) + Bucket referenceBucket = parentBucket.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, tile)); + when(bucketCreator.getOrCreateBucket(parentBucket, tile, rootTileBucket)) .thenReturn(referenceBucket); return referenceBucket; } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java index 02f5295b36..9ac87a25f5 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java @@ -26,7 +26,7 @@ void setUp() { void shouldReturnSetOfFoundResources() { Model model = RDFParser.source("member-with-two-types.ttl").toModel(); - assertThat(referenceBucketiser.bucketise(memberId, model)) + assertThat(referenceBucketiser.createReferences(memberId, model)) .hasSize(2) .contains("https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel") .contains("https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"); @@ -36,7 +36,7 @@ void shouldReturnSetOfFoundResources() { void shouldReturnDefaultBucketString() { Model model = RDFParser.source("member-with-two-types.ttl").toModel(); - assertThat(referenceBucketiser.bucketise("faulty", model)) + assertThat(referenceBucketiser.createReferences("faulty", model)) .hasSize(1) .contains(DEFAULT_BUCKET_STRING); } @@ -45,7 +45,7 @@ void shouldReturnDefaultBucketString() { void when_MemberHasInvalidURI_Then_ReturnOnlyCorrectBucket() { Model model = RDFParser.source("member-with-two-types-faulty.ttl").toModel(); - assertThat(referenceBucketiser.bucketise(memberId, model)) + assertThat(referenceBucketiser.createReferences(memberId, model)) .hasSize(1) .contains("https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"); } @@ -54,7 +54,7 @@ void when_MemberHasInvalidURI_Then_ReturnOnlyCorrectBucket() { void shouldSkipResultsThatAreNotUris() { Model model = RDFParser.source("member-with-string-type.ttl").toModel(); - assertThat(referenceBucketiser.bucketise(memberId, model)).hasSize(1) + assertThat(referenceBucketiser.createReferences(memberId, model)).hasSize(1) .contains(DEFAULT_BUCKET_STRING); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java index 75fa16ae56..405e5650c2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.relations.ReferenceFragmentRelationsAttributer; @@ -13,8 +12,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Optional; - import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.ReferenceFragmentationStrategyWrapper.DEFAULT_FRAGMENTATION_KEY; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator.FRAGMENT_KEY_REFERENCE_ROOT; @@ -28,38 +25,13 @@ class ReferenceBucketCreatorTest { private static final BucketDescriptorPair referenceRootPair = new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, FRAGMENT_KEY_REFERENCE_ROOT); private static final BucketDescriptorPair referencePair = new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, RDF.type.getURI()); private static final BucketDescriptorPair defaultPair = new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, DEFAULT_BUCKET_STRING); - private ReferenceBucketCreator referenceBucketCreator; - - @Mock - private BucketRepository bucketRepository; - @Mock private ReferenceFragmentRelationsAttributer relationsAttributer; @BeforeEach void setUp() { - referenceBucketCreator = - new ReferenceBucketCreator(bucketRepository, relationsAttributer, DEFAULT_FRAGMENTATION_KEY); - } - - @Test - void when_ReferenceFragmentDoesNotExist_NewReferenceFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); - Bucket rootBucket = bucket.createChild(referenceRootPair); - Bucket childBucket = bucket.createChild(referencePair); - - when(bucketRepository.retrieveBucket(viewName, childBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(childBucket); - - Bucket returnedBucket = referenceBucketCreator.getOrCreateBucket(bucket, RDF.type.getURI(), rootBucket); - - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(childBucket); - assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference=%s", RDF.type.getURI()); - verify(bucketRepository).retrieveBucket(viewName, childBucket.getBucketDescriptor()); - verify(bucketRepository).insertBucket(returnedBucket); + referenceBucketCreator = new ReferenceBucketCreator(relationsAttributer, DEFAULT_FRAGMENTATION_KEY); } @Test @@ -67,62 +39,33 @@ void when_ReferenceFragmentDoesExist_RetrievedReferenceFragmentIsReturned() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); Bucket rootBucket = bucket.createChild(referenceRootPair); Bucket referenceBucket = bucket.createChild(referencePair); - - when(bucketRepository.retrieveBucket(viewName, referenceBucket.getBucketDescriptor())).thenReturn(Optional.of(referenceBucket)); + when(relationsAttributer.addRelationFromRootToBottom(rootBucket, bucket.createChild(referencePair))).thenReturn(referenceBucket); Bucket childBucket = referenceBucketCreator.getOrCreateBucket(bucket, RDF.type.getURI(), rootBucket); assertThat(childBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference=%s", RDF.type.getURI()); - verify(bucketRepository).retrieveBucket(viewName, referenceBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addRelationFromRootToBottom(rootBucket, referenceBucket); } @Test void when_RootFragmentDoesNotExist_NewRootFragmentIsCreatedAndSaved() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); - Bucket rootBucket = bucket.createChild(referenceRootPair); - - when(bucketRepository.retrieveBucket(viewName, rootBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(rootBucket); Bucket returnedBucket = referenceBucketCreator.getOrCreateRootBucket(bucket, FRAGMENT_KEY_REFERENCE_ROOT); - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(rootBucket); - verify(bucketRepository).retrieveBucket(viewName, rootBucket.getBucketDescriptor()); - verify(bucketRepository).insertBucket(returnedBucket); + assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference="); + verifyNoInteractions(relationsAttributer); } @Test void when_RootFragmentDoesNotExist_RetrievedRootFragmentIsReturned() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); Bucket rootBucket = bucket.createChild(referenceRootPair); - when(bucketRepository.retrieveBucket(viewName, rootBucket.getBucketDescriptor())).thenReturn(Optional.of(rootBucket)); + when(relationsAttributer.addRelationFromRootToBottom(rootBucket, bucket.createChild(referenceRootPair))).thenReturn(bucket.createChild(referenceRootPair)); Bucket returnedBucket = referenceBucketCreator.getOrCreateBucket(bucket, FRAGMENT_KEY_REFERENCE_ROOT, rootBucket); assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference="); - verify(bucketRepository).retrieveBucket(viewName, rootBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); - } - - @Test - void when_DefaultFragmentDoesNotExist_DefaultFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); - Bucket rootBucket = bucket.createChild(referenceRootPair); - Bucket defaultBucket = bucket.createChild(defaultPair); - - when(bucketRepository.retrieveBucket(viewName, defaultBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(defaultBucket); - - Bucket childBucket = referenceBucketCreator.getOrCreateBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); - - assertThat(childBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(defaultBucket); - verify(bucketRepository).retrieveBucket(viewName, defaultBucket.getBucketDescriptor()); - verify(bucketRepository).insertBucket(childBucket); } @Test @@ -130,14 +73,12 @@ void when_DefaultFragmentDoesExist_RetrievedDefaultFragmentIsReturned() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); Bucket rootBucket = bucket.createChild(referenceRootPair); Bucket defaultBucket = bucket.createChild(defaultPair); - - when(bucketRepository.retrieveBucket(viewName, defaultBucket.getBucketDescriptor())).thenReturn(Optional.of(defaultBucket)); + when(relationsAttributer.addDefaultRelation(bucket, bucket.createChild(defaultPair))).thenReturn(defaultBucket); Bucket childBucket = referenceBucketCreator.getOrCreateBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); assertThat(childBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference=unknown"); - verify(bucketRepository).retrieveBucket(viewName, defaultBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addDefaultRelation(bucket, defaultBucket); } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java index a18a9cf6f2..42c5a8ab58 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java @@ -5,33 +5,27 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import org.apache.jena.datatypes.xsd.XSDDatatype; import org.apache.jena.vocabulary.RDF; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.ReferenceFragmentationStrategyWrapper.DEFAULT_FRAGMENTATION_KEY; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.relations.ReferenceFragmentRelationsAttributer.TREE_REFERENCE_EQUALS_RELATION; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.mockito.Mockito.verify; @ExtendWith(MockitoExtension.class) class ReferenceFragmentRelationsAttributerTest { - private static final String fragmentReference = + private static final String FRAGMENT_REFERENCE = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; private static final ViewName viewName = new ViewName("collectionName", "view"); private static final Bucket parentBucket = new Bucket(BucketDescriptor.empty(), viewName); - @Mock - private ApplicationEventPublisher applicationEventPublisher; - private final String fragmentationPath = RDF.type.getURI(); private ReferenceFragmentRelationsAttributer relationsAttributer; @@ -39,17 +33,14 @@ class ReferenceFragmentRelationsAttributerTest { @BeforeEach void setUp() { relationsAttributer = - new ReferenceFragmentRelationsAttributer(applicationEventPublisher, fragmentationPath, DEFAULT_FRAGMENTATION_KEY); + new ReferenceFragmentRelationsAttributer(fragmentationPath, DEFAULT_FRAGMENTATION_KEY); } @Test void when_ReferenceFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdded() { Bucket rootBucket = createReferenceBuket(""); - Bucket referenceBucket = createReferenceBuket(fragmentReference); - - BucketRelation expectedRelation = new BucketRelation( - rootBucket, - referenceBucket, + Bucket referenceBucket = createReferenceBuket(FRAGMENT_REFERENCE); + TreeRelation expectedRelation = new TreeRelation( TREE_REFERENCE_EQUALS_RELATION, "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel", XSDDatatype.XSDanyURI.getURI(), @@ -58,13 +49,27 @@ void when_ReferenceFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAr relationsAttributer.addRelationFromRootToBottom(rootBucket, referenceBucket); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(expectedRelation)); + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(referenceBucket.withRelations(expectedRelation)); + } + + @Test + void when_UnknownReferenceFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdded() { + Bucket rootBucket = createReferenceBuket(""); + Bucket referenceBucket = createReferenceBuket("unknown"); + + relationsAttributer.addDefaultRelation(rootBucket, referenceBucket); + + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(referenceBucket.withGenericRelation()); } @Test void when_ReferenceFragmentHasNoReference_ThenAnExceptionIsThrown() { Bucket rootBucket = createReferenceBuket(""); - Bucket referenceBucket = parentBucket.createChild(new BucketDescriptorPair("invalid-key", fragmentReference)); + Bucket referenceBucket = parentBucket.createChild(new BucketDescriptorPair("invalid-key", FRAGMENT_REFERENCE)); assertThatExceptionOfType(MissingFragmentValueException.class) .isThrownBy(() -> relationsAttributer.addRelationFromRootToBottom(rootBucket, referenceBucket)) diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml index f9aeeb9f6f..837dc6fa65 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-fragmentisers - 3.3.0 + 3.4.0-SNAPSHOT ldes-fragmentisers-timebased-hierarchical diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java index 8cc5d40f40..255490f20a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; @@ -15,10 +14,8 @@ import org.apache.jena.rdf.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; -import java.util.List; import java.util.Optional; public class HierarchicalTimeBasedFragmentationStrategy extends FragmentationStrategyDecorator { @@ -33,29 +30,27 @@ public class HierarchicalTimeBasedFragmentationStrategy extends FragmentationStr public HierarchicalTimeBasedFragmentationStrategy(FragmentationStrategy fragmentationStrategy, ObservationRegistry observationRegistry, TimeBasedBucketFinder bucketFinder, - ApplicationEventPublisher applicationEventPublisher, TimeBasedConfig config) { - super(fragmentationStrategy, applicationEventPublisher); + super(fragmentationStrategy); this.observationRegistry = observationRegistry; this.bucketFinder = bucketFinder; this.config = config; } @Override - public List addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { final Observation bucketisationObservation = startFragmentationObservation(parentObservation); - Bucket bucket = getFragmentationTimestamp(member.getSubject(), member.getVersionModel()) + Bucket childBucket = getFragmentationTimestamp(member.getSubject(), member.getVersionModel()) .map(timestamp -> bucketFinder.getLowestBucket(parentBucket, timestamp, Granularity.YEAR)) .orElseGet(() -> bucketFinder.getDefaultFragment(parentBucket)); - List members = super.addMemberToBucket(bucket, member, parentObservation); + super.addMemberToBucket(childBucket, member, parentObservation); bucketisationObservation.stop(); - return members; } private Optional getFragmentationTimestamp(String subject, Model memberModel) { - try{ + try { Optional timeStamp = getFragmentationObjectLocalDateTime(memberModel, config.getFragmenterSubjectFilter(), config.getFragmentationPath()); diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java index e7fef9c7c9..5b95f43389 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.services.TimeBasedBucketCreator; @@ -18,22 +17,18 @@ public class HierarchicalTimeBasedFragmentationStrategyWrapper implements Fragme public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, FragmentationStrategy fragmentationStrategy, ConfigProperties fragmentationProperties) { - BucketRepository bucketRepository = applicationContext.getBean(BucketRepository.class); ObservationRegistry observationRegistry = applicationContext.getBean(ObservationRegistry.class); TimeBasedConfig config = createConfig(fragmentationProperties); - TimeBasedRelationsAttributer relationsAttributer = new TimeBasedRelationsAttributer( - applicationContext, config); - TimeBasedBucketCreator bucketCreator = new TimeBasedBucketCreator(bucketRepository, relationsAttributer); + TimeBasedRelationsAttributer relationsAttributer = new TimeBasedRelationsAttributer(config); + TimeBasedBucketCreator bucketCreator = new TimeBasedBucketCreator(relationsAttributer); TimeBasedBucketFinder bucketFinder = new TimeBasedBucketFinder(bucketCreator, config); - return new HierarchicalTimeBasedFragmentationStrategy(fragmentationStrategy, - observationRegistry, bucketFinder, applicationContext, config); + return new HierarchicalTimeBasedFragmentationStrategy(fragmentationStrategy, observationRegistry, bucketFinder, config); } private TimeBasedConfig createConfig(ConfigProperties properties) { return new TimeBasedConfig(properties.getOrDefault(FRAGMENTATION_SUBJECT_FILTER, ".*"), - properties.get(FRAGMENTATION_PATH), Granularity.from(properties.get(MAX_GRANULARITY)), - Boolean.parseBoolean(properties.getOrDefault(LINEAR_TIME_CACHING_ENABLED, "false"))); + properties.get(FRAGMENTATION_PATH), Granularity.from(properties.get(MAX_GRANULARITY))); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java index 4555029e35..f8bf59e5eb 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java @@ -5,15 +5,12 @@ public class TimeBasedConfig { private final String fragmenterSubjectFilter; private final String fragmentationPath; - private final boolean linearTimeCachingEnabled; private final Granularity maxGranularity; - public TimeBasedConfig(String fragmenterSubjectFilter, String fragmentationPath, Granularity maxGranularity, - boolean linearTimeCachingEnabled) { + public TimeBasedConfig(String fragmenterSubjectFilter, String fragmentationPath, Granularity maxGranularity) { this.fragmenterSubjectFilter = fragmenterSubjectFilter; this.fragmentationPath = fragmentationPath; this.maxGranularity = maxGranularity; - this.linearTimeCachingEnabled = linearTimeCachingEnabled; } public String getFragmentationPath() { @@ -27,9 +24,4 @@ public String getFragmenterSubjectFilter() { public Granularity getMaxGranularity() { return maxGranularity; } - - public boolean isLinearTimeCachingEnabled() { - return linearTimeCachingEnabled; - } - } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java index 3aa07d1686..32a53a1773 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java @@ -1,7 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.services; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.model.FragmentationTimestamp; @@ -16,38 +15,31 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.HierarchicalTimeBasedFragmentationStrategy.TIMEBASED_FRAGMENTATION_HIERARCHICAL; public class TimeBasedBucketCreator { - private final BucketRepository bucketRepository; private final TimeBasedRelationsAttributer relationsAttributer; private static final Logger LOGGER = LoggerFactory.getLogger(TimeBasedBucketCreator.class); - public TimeBasedBucketCreator(BucketRepository bucketRepository, TimeBasedRelationsAttributer relationsAttributer) { - this.bucketRepository = bucketRepository; + public TimeBasedBucketCreator(TimeBasedRelationsAttributer relationsAttributer) { this.relationsAttributer = relationsAttributer; } - public Bucket getOrCreateBucket(Bucket parentFragment, - FragmentationTimestamp fragmentationTimestamp, - Granularity granularity) { - return getOrCreateBucket(parentFragment, fragmentationTimestamp.getTimeValueForGranularity(granularity), granularity); + public Bucket createBucket(Bucket parentBucket, + FragmentationTimestamp fragmentationTimestamp, + Granularity granularity) { + return createBucket(parentBucket, fragmentationTimestamp.getTimeValueForGranularity(granularity), granularity); } - public Bucket getOrCreateBucket(Bucket parentBucket, String timeValue, Granularity granularity) { + public Bucket createBucket(Bucket parentBucket, String timeValue, Granularity granularity) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(granularity.getValue(), timeValue); - return bucketRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = bucketRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - addRelationToParent(parentBucket, childBucket); - logBucketisation(parentBucket, childBucket); - return childBucket; - }); + final Bucket childBucket = parentBucket.createChild(childDescriptorPair); + logBucketisation(parentBucket, childBucket); + return addRelationToParent(parentBucket, childBucket); } - private void addRelationToParent(Bucket parentBucket, Bucket childBucket) { - if(isDefaultBucket(childBucket)) { - relationsAttributer.addDefaultRelation(parentBucket, childBucket); + private Bucket addRelationToParent(Bucket parentBucket, Bucket childBucket) { + if (isDefaultBucket(childBucket)) { + return relationsAttributer.addDefaultRelation(parentBucket, childBucket); } else { - relationsAttributer.addInBetweenRelation(parentBucket, childBucket); + return relationsAttributer.addInBetweenRelation(parentBucket, childBucket); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java index d0f48a43f2..f20eacdea2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java @@ -22,16 +22,15 @@ public Bucket getLowestBucket(Bucket parentFragment, FragmentationTimestamp frag return parentFragment; } return getLowestBucket( - bucketCreator.getOrCreateBucket(parentFragment, fragmentationTimestamp, granularity), + bucketCreator.createBucket(parentFragment, fragmentationTimestamp, granularity), fragmentationTimestamp, granularity.getChild()); } public Bucket getDefaultFragment(Bucket rootFragment) { - return bucketCreator.getOrCreateBucket(rootFragment, DEFAULT_BUCKET_STRING, Granularity.YEAR); + return bucketCreator.createBucket(rootFragment, DEFAULT_BUCKET_STRING, Granularity.YEAR); } private boolean isLowest(Bucket bucket) { - return bucket.getBucketDescriptorPairs().stream() - .anyMatch(descriptorPair -> descriptorPair.key().equals(config.getMaxGranularity().getValue())); + return bucket.getValueForKey(config.getMaxGranularity().getValue()).isPresent(); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java index 61c0dab5c0..3e1d2cb2d2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java @@ -1,85 +1,60 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.TimeBasedLinearCachingTriggered; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.model.FragmentationTimestamp; import org.jetbrains.annotations.NotNull; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; import java.util.Arrays; import java.util.Map; +import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.TimeBasedConstants.*; public class TimeBasedRelationsAttributer implements RelationsAttributer { - private final ApplicationEventPublisher applicationEventPublisher; - private final TimeBasedConfig config; - public TimeBasedRelationsAttributer(ApplicationEventPublisher applicationEventPublisher, TimeBasedConfig config) { - this.applicationEventPublisher = applicationEventPublisher; + public TimeBasedRelationsAttributer(TimeBasedConfig config) { this.config = config; } - public void addInBetweenRelation(Bucket parentBucket, Bucket childBucket) { + public Bucket addInBetweenRelation(Bucket parentBucket, Bucket childBucket) { FragmentationTimestamp timestamp = timestampFromFragmentPairs(childBucket); - BucketRelation parentGteRelation = new BucketRelation( - parentBucket, - childBucket, - TREE_GTE_RELATION, - timestamp.getTime().toString(), - XSD_DATETIME, - config.getFragmentationPath() - ); - BucketRelation parentLtRelation = new BucketRelation( - parentBucket, - childBucket, - TREE_LT_RELATION, - timestamp.getLtBoundary().toString(), - XSD_DATETIME, - config.getFragmentationPath() - ); - saveRelation(parentGteRelation, timestamp.getNextUpdateTs()); - saveRelation(parentLtRelation, timestamp.getNextUpdateTs()); - } - - public void addDefaultRelation(Bucket parentBucket, Bucket childBucket) { - final BucketRelation defaultRelation = BucketRelation.createGenericRelation(parentBucket, childBucket); - saveRelation(defaultRelation, null); + return parentBucket.addChildBucket(childBucket.withRelations(createTimeBasedRelations(timestamp))); } - private void saveRelation(BucketRelation bucketRelation, LocalDateTime nextUpdateTs) { - if (config.isLinearTimeCachingEnabled()) { - applicationEventPublisher.publishEvent(new TimeBasedLinearCachingTriggered(bucketRelation.fromBucket().getBucketId(), nextUpdateTs)); - } - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); - + public Bucket addDefaultRelation(Bucket parentBucket, Bucket childBucket) { + return parentBucket.addChildBucket(childBucket.withGenericRelation()); } private FragmentationTimestamp timestampFromFragmentPairs(Bucket bucket) { - Map timeMap = bucket.getBucketDescriptorPairs().stream() - .filter(descriptorPair -> Arrays.stream(Granularity.values()).map(Granularity::getValue) - .anyMatch(t -> t.equals(descriptorPair.key()))) - .collect(Collectors.toMap(BucketDescriptorPair::key, pair -> Integer.parseInt(pair.value()))); + Map> timeMap = Arrays.stream(Granularity.values()) + .map(Granularity::getValue) + .collect(Collectors.toMap(Function.identity(), key -> bucket.getValueForKey(key).map(Integer::parseInt))); return createTimestampFromMap(timeMap); } - private static @NotNull FragmentationTimestamp createTimestampFromMap(Map timeMap) { - LocalDateTime time = LocalDateTime.of(timeMap.getOrDefault(Granularity.YEAR.getValue(), 0), - timeMap.getOrDefault(Granularity.MONTH.getValue(), 1), - timeMap.getOrDefault(Granularity.DAY.getValue(), 1), - timeMap.getOrDefault(Granularity.HOUR.getValue(), 0), - timeMap.getOrDefault(Granularity.MINUTE.getValue(), 0), - timeMap.getOrDefault(Granularity.SECOND.getValue(), 0)); - return new FragmentationTimestamp(time, Granularity.fromIndex(timeMap.size() - 1)); + private TreeRelation[] createTimeBasedRelations(FragmentationTimestamp timestamp) { + return new TreeRelation[]{ + new TreeRelation(TREE_GTE_RELATION, timestamp.getTime().toString(), XSD_DATETIME, config.getFragmentationPath()), + new TreeRelation(TREE_LT_RELATION, timestamp.getLtBoundary().toString(), XSD_DATETIME, config.getFragmentationPath()), + }; + } + + private static @NotNull FragmentationTimestamp createTimestampFromMap(Map> timeMap) { + LocalDateTime time = LocalDateTime.of(timeMap.get(Granularity.YEAR.getValue()).orElse(0), + timeMap.get(Granularity.MONTH.getValue()).orElse(1), + timeMap.get(Granularity.DAY.getValue()).orElse(1), + timeMap.get(Granularity.HOUR.getValue()).orElse(0), + timeMap.get(Granularity.MINUTE.getValue()).orElse(0), + timeMap.get(Granularity.SECOND.getValue()).orElse(0)); + return new FragmentationTimestamp(time, Granularity.fromIndex((int) (timeMap.values().stream().filter(Optional::isPresent).count() - 1))); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java index 9bfa454da7..466855288e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java @@ -30,37 +30,24 @@ class HierarchicalTimeBasedFragmentationStrategyTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - private static final Bucket CHILD_BUCKET = new Bucket(new BucketDescriptor(List.of(new BucketDescriptorPair("is", "child"))), VIEW_NAME); private static final LocalDateTime TIME = LocalDateTime.of(2023, 1, 1, 0, 0, 0); private static final Granularity GRANULARITY = Granularity.SECOND; private static final EventStreamProperties EVENT_STREAM_PROPERTIES = new EventStreamProperties("collectionName", "versionOf", "timestampPath", false); private HierarchicalTimeBasedFragmentationStrategy fragmentationStrategy; private FragmentationStrategy decoratedFragmentationStrategy; private TimeBasedBucketFinder bucketFinder; + private Bucket parentBucket; + private Bucket childBucket; @BeforeEach void setUp() { - TimeBasedConfig config = new TimeBasedConfig(".*", "http://purl.org/dc/terms/created", GRANULARITY, false); + TimeBasedConfig config = new TimeBasedConfig(".*", "http://purl.org/dc/terms/created", GRANULARITY); bucketFinder = mock(TimeBasedBucketFinder.class); decoratedFragmentationStrategy = mock(FragmentationStrategy.class); fragmentationStrategy = new HierarchicalTimeBasedFragmentationStrategy(decoratedFragmentationStrategy, - ObservationRegistry.create(), bucketFinder, mock(), config); - } - - @Test - void when_FragmentationCalled_Then_FunctionsAreCalled() { - Model model = loadModel("member_with_created_property.nq"); - FragmentationMember fragmentationMember = new FragmentationMember(1, "subject", "versionOf", TIME, EVENT_STREAM_PROPERTIES, model); - FragmentationTimestamp fragmentationTimestamp = new FragmentationTimestamp(TIME, GRANULARITY); - when(bucketFinder.getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR)) - .thenReturn(CHILD_BUCKET); - - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, fragmentationMember, mock()); - - InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR); - inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(CHILD_BUCKET), any(), any()); + ObservationRegistry.create(), bucketFinder, config); + parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + childBucket = new Bucket(new BucketDescriptor(List.of(new BucketDescriptorPair("is", "child"))), VIEW_NAME); } @Test @@ -68,44 +55,26 @@ void when_BucketisationCalled_Then_FunctionsAreCalled() { Model model = loadModel("member_with_created_property.nq"); FragmentationMember member = new FragmentationMember(1, "subject", "versionOf", TIME, EVENT_STREAM_PROPERTIES, model); FragmentationTimestamp fragmentationTimestamp = new FragmentationTimestamp(TIME, GRANULARITY); - when(bucketFinder.getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR)) - .thenReturn(CHILD_BUCKET); - - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); - - InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR); - inOrder.verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(CHILD_BUCKET), any(), - any(Observation.class)); - } - - @Test - void when_FragmentationCalledForMemberWithMissingTimestamp_Then_FunctionsAreCalled() { - FragmentationMember member = mock(FragmentationMember.class); - when(bucketFinder.getDefaultFragment(PARENT_BUCKET)).thenReturn(CHILD_BUCKET); + when(bucketFinder.getLowestBucket(parentBucket, fragmentationTimestamp, Granularity.YEAR)) + .thenReturn(childBucket); - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock()); + fragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getDefaultFragment(PARENT_BUCKET); - inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(CHILD_BUCKET), any(), any()); + inOrder.verify(bucketFinder).getLowestBucket(parentBucket, fragmentationTimestamp, Granularity.YEAR); + inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(childBucket), any(), any(Observation.class)); } @Test void when_BucketisationCalledForMemberWithMissingTimestamp_Then_FunctionsAreCalled() { FragmentationMember member = mock(FragmentationMember.class); - when(bucketFinder.getDefaultFragment(PARENT_BUCKET)) - .thenReturn(CHILD_BUCKET); + when(bucketFinder.getDefaultFragment(parentBucket)).thenReturn(childBucket); - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, - mock(Observation.class)); + fragmentationStrategy.addMemberToBucket(parentBucket, member, mock()); InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getDefaultFragment(PARENT_BUCKET); - inOrder.verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(CHILD_BUCKET), any(), - any(Observation.class)); + inOrder.verify(bucketFinder).getDefaultFragment(parentBucket); + inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(childBucket), any(), any(Observation.class)); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java index 60b4aaee83..81b372b314 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; @@ -12,7 +11,6 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.Optional; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static org.assertj.core.api.Assertions.assertThat; @@ -21,73 +19,44 @@ class TimeBasedBucketCreatorTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final BucketDescriptorPair timePair = new BucketDescriptorPair(Granularity.YEAR.getValue(), "2023"); - private static final Bucket PARENT = new Bucket(new BucketDescriptor(List.of(timePair)), VIEW_NAME); - private static final Bucket ROOT = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + private Bucket parent; + private Bucket root; private static final FragmentationTimestamp TIME = new FragmentationTimestamp(LocalDateTime.of(2023, 1, 1, 0, 0, 0), Granularity.MONTH); - private BucketRepository bucketRepository; private TimeBasedRelationsAttributer relationsAttributer; private TimeBasedBucketCreator bucketCreator; @BeforeEach void setUp() { - bucketRepository = mock(BucketRepository.class); relationsAttributer = mock(TimeBasedRelationsAttributer.class); - bucketCreator = new TimeBasedBucketCreator(bucketRepository, relationsAttributer); + bucketCreator = new TimeBasedBucketCreator(relationsAttributer); + parent = new Bucket(new BucketDescriptor(List.of(timePair)), VIEW_NAME); + root = Bucket.createRootBucketForView(VIEW_NAME); } @Test - void when_FragmentDoesNotExist_Then_NewFragmentIsCreated() { - final BucketDescriptor expectedBucketDescriptor = new BucketDescriptor(List.of(timePair, new BucketDescriptorPair(Granularity.MONTH.getValue(), "01"))); - final Bucket expectedChild = new Bucket(expectedBucketDescriptor, VIEW_NAME); - when(bucketRepository.retrieveBucket(VIEW_NAME, expectedBucketDescriptor)).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(expectedChild); + void test_GetOrCreateInBetweenBucket() { + final BucketDescriptor childDescriptor = new BucketDescriptor(List.of(timePair, new BucketDescriptorPair(Granularity.MONTH.getValue(), "01"))); + when(relationsAttributer.addInBetweenRelation(parent, new Bucket(childDescriptor, VIEW_NAME))) + .thenReturn(new Bucket(childDescriptor, VIEW_NAME)); - Bucket child = bucketCreator.getOrCreateBucket(PARENT, TIME, Granularity.MONTH); + Bucket child = bucketCreator.createBucket(parent, TIME, Granularity.MONTH); - assertThat(child) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(expectedChild) - .extracting(Bucket::getBucketDescriptor) - .isEqualTo(expectedBucketDescriptor); - verify(bucketRepository).retrieveBucket(VIEW_NAME, expectedBucketDescriptor); - verify(relationsAttributer).addInBetweenRelation(PARENT, child); - verify(bucketRepository).insertBucket(child); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addInBetweenRelation(eq(parent), any()); + assertThat(child.getBucketDescriptor()).isEqualTo(childDescriptor); } @Test - void when_FragmentDoesNotExistAndIsDefaultFragment_Then_NewFragmentIsCreated() { - BucketDescriptor expectedBucketDescriptor = new BucketDescriptor( + void test_GetOrCreateDefaultBucket() { + BucketDescriptor childDescriptor = new BucketDescriptor( List.of(new BucketDescriptorPair(Granularity.YEAR.getValue(), DEFAULT_BUCKET_STRING))); - final Bucket expectedChild = new Bucket(expectedBucketDescriptor, VIEW_NAME); - when(bucketRepository.retrieveBucket(VIEW_NAME, expectedBucketDescriptor)).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(expectedChild); + when(relationsAttributer.addDefaultRelation(root, new Bucket(childDescriptor, VIEW_NAME))) + .thenReturn(new Bucket(childDescriptor, VIEW_NAME)); - Bucket child = bucketCreator.getOrCreateBucket(new Bucket(BucketDescriptor.empty(), VIEW_NAME), DEFAULT_BUCKET_STRING, Granularity.YEAR); + Bucket child = bucketCreator.createBucket(root, DEFAULT_BUCKET_STRING, Granularity.YEAR); - assertThat(child) - .describedAs("Child instance must be the same, to assure the bucket inserted into the db is returned") - .isSameAs(expectedChild) - .extracting(Bucket::getBucketDescriptor) - .isEqualTo(expectedBucketDescriptor); - verify(bucketRepository).retrieveBucket(VIEW_NAME, expectedBucketDescriptor); - verify(relationsAttributer).addDefaultRelation(ROOT, child); - verify(bucketRepository).insertBucket(child); - verifyNoMoreInteractions(bucketRepository); - } - - @Test - void when_FragmentDoesExist_Then_FragmentIsRetrieved() { - Bucket expectedChild = PARENT.createChild(new BucketDescriptorPair(Granularity.MONTH.getValue(), "01")); - when(bucketRepository.retrieveBucket(VIEW_NAME, expectedChild.getBucketDescriptor())).thenReturn(Optional.of(expectedChild)); - - Bucket child = bucketCreator.getOrCreateBucket(PARENT, TIME, Granularity.MONTH); - - assertThat(child.getBucketDescriptorAsString()).isEqualTo(expectedChild.getBucketDescriptorAsString()); - verify(bucketRepository).retrieveBucket(VIEW_NAME, expectedChild.getBucketDescriptor()); - verifyNoInteractions(relationsAttributer); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addDefaultRelation(eq(root), any()); + assertThat(child.getBucketDescriptor()).isEqualTo(childDescriptor); } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java index 14d87fe46c..98acb2d5a2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java @@ -26,7 +26,7 @@ class TimeBasedBucketFinderTest { @BeforeEach void setUp() { - TimeBasedConfig config = new TimeBasedConfig(".*", "", Granularity.DAY, false); + TimeBasedConfig config = new TimeBasedConfig(".*", "", Granularity.DAY); bucketCreator = mock(TimeBasedBucketCreator.class); bucketFinder = new TimeBasedBucketFinder(bucketCreator, config); @@ -43,9 +43,9 @@ void when_GetLowestIsCalled_Then_ReturnExpectedBucket() { Bucket yearBucket = PARENT.createChild(new BucketDescriptorPair(Granularity.YEAR.getValue(), "2023")); Bucket monthBucket = yearBucket.createChild(new BucketDescriptorPair(Granularity.MONTH.getValue(), "01")); Bucket dayBucket = monthBucket.createChild(new BucketDescriptorPair(Granularity.DAY.getValue(), "01")); - when(bucketCreator.getOrCreateBucket(PARENT, TIME, Granularity.YEAR)).thenReturn(yearBucket); - when(bucketCreator.getOrCreateBucket(yearBucket, TIME, Granularity.MONTH)).thenReturn(monthBucket); - when(bucketCreator.getOrCreateBucket(monthBucket, TIME, Granularity.DAY)).thenReturn(dayBucket); + when(bucketCreator.createBucket(PARENT, TIME, Granularity.YEAR)).thenReturn(yearBucket); + when(bucketCreator.createBucket(yearBucket, TIME, Granularity.MONTH)).thenReturn(monthBucket); + when(bucketCreator.createBucket(monthBucket, TIME, Granularity.DAY)).thenReturn(dayBucket); Bucket actual = bucketFinder.getLowestBucket(PARENT, TIME, Granularity.YEAR); @@ -56,7 +56,7 @@ void when_GetLowestIsCalled_Then_ReturnExpectedBucket() { void when_GetDefaultIsCalled_Then_ReturnExpectedFragment() { bucketFinder.getDefaultFragment(PARENT); - verify(bucketCreator).getOrCreateBucket(PARENT, DEFAULT_BUCKET_STRING, Granularity.YEAR); + verify(bucketCreator).createBucket(PARENT, DEFAULT_BUCKET_STRING, Granularity.YEAR); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java index 4ccc1bba63..72e920acaa 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java @@ -4,97 +4,66 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.TimeBasedConstants.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; class TimeBasedRelationsAttributerTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final BucketDescriptorPair timePair = new BucketDescriptorPair(Granularity.YEAR.getValue(), "2023"); private static final BucketDescriptorPair monthPair = new BucketDescriptorPair(Granularity.MONTH.getValue(), "02"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.of(timePair), VIEW_NAME); + private Bucket parentBucket; private TimeBasedRelationsAttributer relationsAttributer; - private ApplicationEventPublisher applicationEventPublisher; private TimeBasedConfig config; @BeforeEach void setUp() { - applicationEventPublisher = mock(ApplicationEventPublisher.class); - config = new TimeBasedConfig(".*", "", Granularity.SECOND, false); - relationsAttributer = new TimeBasedRelationsAttributer(applicationEventPublisher, config); + config = new TimeBasedConfig(".*", "", Granularity.SECOND); + relationsAttributer = new TimeBasedRelationsAttributer(config); + parentBucket = new Bucket(BucketDescriptor.of(timePair), VIEW_NAME); } @Test void when_RelationNotPresent_AndCachingDisabled_ThenRelationIsAdded_NextUpdateTsIsNotSet_ChildrenStayMutable() { - Bucket child = PARENT_BUCKET.createChild(monthPair); + Bucket child = parentBucket.createChild(monthPair); - BucketRelation gteRelation = new BucketRelation( - PARENT_BUCKET, - child, + TreeRelation gteRelation = new TreeRelation( TREE_GTE_RELATION, LocalDateTime.of(2023,2,1,0,0).toString(), XSD_DATETIME, config.getFragmentationPath()); - BucketRelation ltRelation = new BucketRelation( - PARENT_BUCKET, - child, + TreeRelation ltRelation = new TreeRelation( TREE_LT_RELATION, LocalDateTime.of(2023,3,1,0,0).toString(), XSD_DATETIME, config.getFragmentationPath()); - relationsAttributer.addInBetweenRelation(PARENT_BUCKET, child); + relationsAttributer.addInBetweenRelation(parentBucket, child); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(gteRelation)); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(ltRelation)); - } - - @Test - void when_RelationNotPresent_AndCachingEnabled_ThenRelationIsAdded_NextUpdateTsIsSet_AndChildrenBecomeImmutable() { - config = new TimeBasedConfig(".*", "", Granularity.SECOND, true); - relationsAttributer = new TimeBasedRelationsAttributer(applicationEventPublisher, config); - Bucket child = PARENT_BUCKET.createChild(monthPair); - - BucketRelation gteRelation = new BucketRelation( - PARENT_BUCKET, - child, - TREE_GTE_RELATION, - LocalDateTime.of(2023,2,1,0,0).toString(), - XSD_DATETIME, - config.getFragmentationPath()); - BucketRelation ltRelation = new BucketRelation( - PARENT_BUCKET, - child, - TREE_LT_RELATION, - LocalDateTime.of(2023,3,1,0,0).toString(), - XSD_DATETIME, - config.getFragmentationPath()); - - relationsAttributer.addInBetweenRelation(PARENT_BUCKET, child); + assertThat(parentBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder( + child.withRelations(gteRelation, ltRelation) + ); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(gteRelation)); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(ltRelation)); } @Test void when_RelationNotPresent_Then_AddDefaultRelation() { - Bucket child = PARENT_BUCKET.createChild(monthPair); - BucketRelation expected = BucketRelation.createGenericRelation(PARENT_BUCKET, child); + Bucket child = parentBucket.createChild(monthPair); - relationsAttributer.addDefaultRelation(PARENT_BUCKET, child); + relationsAttributer.addDefaultRelation(parentBucket, child); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(expected)); + assertThat(parentBucket.getChildren()) + .containsExactly(child.withGenericRelation()); } } diff --git a/ldes-fragmentisers/pom.xml b/ldes-fragmentisers/pom.xml index 0ec6d6e2c8..e60483037e 100644 --- a/ldes-fragmentisers/pom.xml +++ b/ldes-fragmentisers/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-fragmentisers diff --git a/ldes-server-admin/pom.xml b/ldes-server-admin/pom.xml index 181453806f..47fecf78a9 100644 --- a/ldes-server-admin/pom.xml +++ b/ldes-server-admin/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index d050d5121a..0bce3852db 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-application diff --git a/ldes-server-compaction/pom.xml b/ldes-server-compaction/pom.xml index 9a14a5ea20..ed5971a3a4 100644 --- a/ldes-server-compaction/pom.xml +++ b/ldes-server-compaction/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-compaction diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java b/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java index 8901e7954c..0b79fad278 100644 --- a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java +++ b/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyImpl; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,7 +9,7 @@ public class FragmentationConfigCompaction { @Bean("compactionFragmentation") - public FragmentationStrategy compactionFragmentationStrategy(ApplicationEventPublisher eventPublisher) { + public FragmentationStrategy compactionFragmentationStrategy() { return new FragmentationStrategyImpl(); } } diff --git a/ldes-server-domain/pom.xml b/ldes-server-domain/pom.xml index 62afe59bf6..90cc3253e6 100644 --- a/ldes-server-domain/pom.xml +++ b/ldes-server-domain/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-domain diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java deleted file mode 100644 index 7e2d49830e..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.compaction; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; - -import java.util.List; - -public record FragmentsCompactedEvent(List compactedFragments) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java deleted file mode 100644 index 31600aa51c..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; - -import java.util.Set; - -public record BulkFragmentDeletedEvent(Set ldesFragmentIdentifiers) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java deleted file mode 100644 index 66e13a3590..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java +++ /dev/null @@ -1,6 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import java.util.List; - -public record BulkMemberAllocatedEvent(List membersOfCompactedFragments, String collectionName, String viewName, String fragmentId) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java deleted file mode 100644 index c80c049aab..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java +++ /dev/null @@ -1,5 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - - -public record MemberAllocatedEvent(String memberId, String collectionName, String viewName, String fragmentId) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java deleted file mode 100644 index 3c95ea824e..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java +++ /dev/null @@ -1,6 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import java.time.LocalDateTime; - -public record TimeBasedLinearCachingTriggered(long bucketId, LocalDateTime nextUpdateTs) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java deleted file mode 100644 index 521dca13e9..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java +++ /dev/null @@ -1,12 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -/** - * This event is published as a response to {@link ViewInitializationEvent} or - * to {@link ViewAddedEvent}. This will trigger rebucketisation of the member for a view. - */ -public record ViewNeedsRebucketisationEvent(ViewName viewName) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java deleted file mode 100644 index 551d6d4b75..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java +++ /dev/null @@ -1,9 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest; - -import java.time.LocalDateTime; -import java.util.List; - -public record MembersIngestedEvent(String collectionName, List members) { - public record MemberProperties(String id, String versionOf, LocalDateTime timestamp) { - } -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java deleted file mode 100644 index 0a229a85f2..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java +++ /dev/null @@ -1,32 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions; - -@SuppressWarnings("java:S3740") -public class DataConversionException extends RuntimeException { - private final Class conversionClass; - private final String operation; - - public DataConversionException(Class conversionClass, boolean serialising, Exception e) { - super(e); - this.conversionClass = conversionClass; - this.operation = serialising ? "serializing" : "deserializing"; - } - - private DataConversionException(Class conversionClass, String operation, Exception e) { - super(e); - this.conversionClass = conversionClass; - this.operation = operation; - } - - @Override - public String getMessage() { - return "Failed %s class %s failed.".formatted(operation, conversionClass); - } - - public static DataConversionException serializationFailed(Class conversionClass, Exception e) { - return new DataConversionException(conversionClass, "serializing", e); - } - - public static DataConversionException deserializationFailed(Class conversionClass, Exception e) { - return new DataConversionException(conversionClass, "deserializing", e); - } -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java deleted file mode 100644 index 289c16a015..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; - -public record FragmentPair(String fragmentKey, String fragmentValue) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java deleted file mode 100644 index c3d02660fa..0000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java +++ /dev/null @@ -1,13 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; - -public record FragmentSequence(ViewName viewName, long sequenceNr) { - - /** - * Creates a sequence with a negative number. The ingest module starts sequencing from 0. - * This negative sequenceNr indicates that this view has never been processed for fragmentation. - */ - public static FragmentSequence createNeverProcessedSequence(ViewName viewName) { - return new FragmentSequence(viewName, -1); - } - -} diff --git a/ldes-server-domain/src/main/java/module-info.java b/ldes-server-domain/src/main/java/module-info.java index f8f372d29c..ec845fd5ae 100644 --- a/ldes-server-domain/src/main/java/module-info.java +++ b/ldes-server-domain/src/main/java/module-info.java @@ -7,11 +7,8 @@ exports be.vlaanderen.informatievlaanderen.ldes.server.domain.rest; // Events - exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.retention; - exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin; - exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.compaction; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.services; requires spring.web; diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 427e378656..1bd4c3efca 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT pom diff --git a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml index ecceeaf648..af0cdc88e7 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0 + 3.4.0-SNAPSHOT postgres-admin-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml index baedec41e7..10572b6033 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0 + 3.4.0-SNAPSHOT postgres-fragmentation-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java index f5d186581a..5bf5169fd9 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java @@ -8,10 +8,12 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper.BucketMapper; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.BucketEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -19,32 +21,36 @@ public class BucketPostgresRepository implements BucketRepository { private final ViewEntityRepository viewEntityRepository; private final BucketEntityRepository bucketEntityRepository; + private final NamedParameterJdbcTemplate jdbcTemplate; - public BucketPostgresRepository(ViewEntityRepository viewEntityRepository, BucketEntityRepository bucketEntityRepository) { + public BucketPostgresRepository(ViewEntityRepository viewEntityRepository, BucketEntityRepository bucketEntityRepository, NamedParameterJdbcTemplate jdbcTemplate) { this.viewEntityRepository = viewEntityRepository; this.bucketEntityRepository = bucketEntityRepository; - } - - @Override - public Optional retrieveBucket(ViewName viewName, BucketDescriptor bucketDescriptor) { - return bucketEntityRepository - .findBucketEntityByBucketDescriptor(viewName.asString(), bucketDescriptor.asDecodedString()) - .map(BucketMapper::fromProjection); + this.jdbcTemplate = jdbcTemplate; } @Override @Transactional public Bucket insertBucket(Bucket bucket) { + final String sql = """ + INSERT INTO pages (bucket_id, expiration, partial_url) + VALUES (:bucketId, NULL, :partialUrl) + ON CONFLICT DO NOTHING + """; ViewEntity view = viewEntityRepository.findByViewName(bucket.getViewName().getCollectionName(), bucket.getViewName().getViewName()) .orElseThrow(); BucketEntity bucketEntity = new BucketEntity(view, bucket.getBucketDescriptorAsString()); bucketEntity = bucketEntityRepository.save(bucketEntity); + long bucketId = Objects.requireNonNull(bucketEntity.getBucketId()); + jdbcTemplate.update(sql, Map.of("bucketId", bucketId, "partialUrl", bucket.createPartialUrl())); return new Bucket( - Objects.requireNonNull(bucketEntity.getBucketId()), + bucketId, bucket.getBucketDescriptor(), - bucket.getViewName() + bucket.getViewName(), + List.of(), + 0 ); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java new file mode 100644 index 0000000000..0fac843a46 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +@Component +public class BucketisationItemWriter implements ItemWriter { + private final ItemWriter bucketItemWriter; + private final ItemWriter pageItemWriter; + private final ItemWriter bucketisedMemberItemWriter; + private final ItemWriter bucketRelationWriter; + + public BucketisationItemWriter(ItemWriter bucketItemWriter, + ItemWriter pageItemWriter, + ItemWriter bucketisedMemberItemWriter, + ItemWriter bucketRelationWriter) { + this.bucketItemWriter = bucketItemWriter; + this.pageItemWriter = pageItemWriter; + this.bucketisedMemberItemWriter = bucketisedMemberItemWriter; + this.bucketRelationWriter = bucketRelationWriter; + } + + @Override + public void write(Chunk rootBucketChunk) throws Exception { + for(var rootbucket : rootBucketChunk) { + final Chunk flatBucketChunk = new Chunk<>(rootbucket.getBucketTree()); + bucketItemWriter.write(flatBucketChunk); + pageItemWriter.write(flatBucketChunk); + bucketRelationWriter.write(extractAllBucketRelations(rootbucket)); + bucketisedMemberItemWriter.write(extractAllMembers(flatBucketChunk)); + } + } + + private static Chunk extractAllMembers(Chunk flatBucketChunk) { + return flatBucketChunk.getItems().stream() + .map(Bucket::getMember) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(new ChunkCollector<>()); + } + + private static Chunk extractAllBucketRelations(Bucket rootbucket) { + return rootbucket.getBucketTree().stream() + .flatMap(bucket -> bucket.getChildRelations().stream()) + .distinct() + .collect(new ChunkCollector<>()); + } + +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java deleted file mode 100644 index eae657569c..0000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java +++ /dev/null @@ -1,54 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.database.JdbcBatchItemWriter; -import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import javax.sql.DataSource; -import java.util.List; - -@Component -public class BucketisedMemberWriter implements ItemWriter> { - private final JdbcBatchItemWriter delegateWriter; - - public BucketisedMemberWriter(JdbcBatchItemWriter delegateWriter) { - this.delegateWriter = delegateWriter; - } - - @Override - public void write(Chunk> chunk) throws Exception { - Chunk bucketisedMembers = new Chunk<>(chunk.getItems() - .stream() - .flatMap(List::stream) - .toList()); - - if (!bucketisedMembers.isEmpty()) { - delegateWriter.write(bucketisedMembers); - } - } - - @Configuration - public static class BatchBucketWriterConfig { - private static final String SQL = """ - INSERT INTO page_members (bucket_id, member_id) - VALUES (?, ?) - """; - - @Bean - JdbcBatchItemWriter batchBucketWriter(DataSource dataSource) { - return new JdbcBatchItemWriterBuilder() - .dataSource(dataSource) - .sql(SQL) - .itemPreparedStatementSetter((item, ps) -> { - ps.setLong(1, item.bucketId()); - ps.setLong(2, item.memberId()); - }) - .build(); - } - } -} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java new file mode 100644 index 0000000000..3d148b8c10 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java @@ -0,0 +1,40 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk; + +import com.google.common.collect.Streams; +import org.springframework.batch.item.Chunk; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; + +public class ChunkCollector implements Collector, Chunk> { + @Override + public Supplier> supplier() { + return Chunk::new; + } + + @Override + public BiConsumer, T> accumulator() { + return Chunk::add; + } + + @Override + public BinaryOperator> combiner() { + return (left, right) -> new Chunk<>(Streams.concat(left.getItems().stream(), right.getItems().stream()).toList()); + } + + @Override + public Function, Chunk> finisher() { + return Function.identity(); + } + + @Override + public Set characteristics() { + return Collections.unmodifiableSet(EnumSet.of(Collector.Characteristics.UNORDERED, Collector.Characteristics.IDENTITY_FINISH)); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java new file mode 100644 index 0000000000..fb4337a8b6 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java @@ -0,0 +1,49 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Objects; + +@Component +public class BucketItemWriter implements ItemWriter { + private static final String SQL = """ + INSERT INTO buckets (bucket, view_id) + SELECT :bucket, view_id + FROM views v + JOIN collections c USING (collection_id) + WHERE c.name = :collectionName + AND v.name = :viewName + ON CONFLICT (bucket, view_id) DO UPDATE SET bucket_id = buckets.bucket_id + """; + + private final NamedParameterJdbcTemplate jdbcTemplate; + + public BucketItemWriter(NamedParameterJdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public void write(Chunk chunk) { + for (var bucket : chunk) { + GeneratedKeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.update( + SQL, + new MapSqlParameterSource(Map.of( + "bucket", bucket.getBucketDescriptorAsString(), + "collectionName", bucket.getViewName().getCollectionName(), + "viewName", bucket.getViewName().getViewName() + )), + keyHolder, + new String[]{"bucket_id"} + ); + bucket.setBucketId(Objects.requireNonNull(keyHolder.getKey()).longValue()); + } + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java new file mode 100644 index 0000000000..43b8924987 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import javax.sql.DataSource; +import java.util.Map; + +@Configuration +public class BucketPageItemWriterConfig { + private static final String SQL = """ + INSERT INTO pages (bucket_id, expiration, partial_url) + VALUES (:bucketId, NULL, :partialUrl) + ON CONFLICT DO NOTHING + """; + + @Bean + ItemWriter pageItemWriter(DataSource dataSource) { + return new JdbcBatchItemWriterBuilder() + .dataSource(dataSource) + .sql(SQL) + .assertUpdates(false) + .itemSqlParameterSourceProvider(item -> new MapSqlParameterSource(Map.of( + "bucketId", item.getBucketId(), + "partialUrl", item.createPartialUrl() + ))) + .build(); + + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java new file mode 100644 index 0000000000..900c9acc15 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java @@ -0,0 +1,32 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import javax.sql.DataSource; +import java.util.Map; + +@Configuration +public class BucketisedMemberItemWriterConfig { + private static final String SQL = """ + INSERT INTO page_members (bucket_id, member_id, view_id) + SELECT :bucketId, :memberId, view_id + FROM buckets WHERE bucket_id = :bucketId + """; + + @Bean + public ItemWriter bucketisedMemberWriter(DataSource dataSource) { + return new JdbcBatchItemWriterBuilder() + .dataSource(dataSource) + .sql(SQL) + .itemSqlParameterSourceProvider(item -> new MapSqlParameterSource(Map.of( + "bucketId", item.bucketId(), + "memberId", item.memberId() + ))) + .build(); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java new file mode 100644 index 0000000000..c1ddd8288e --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java @@ -0,0 +1,37 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import javax.sql.DataSource; +import java.util.Map; + +@Configuration +public class PageRelationItemWriterConfig { + + @Bean + public ItemWriter bucketRelationItemWriter(DataSource dataSource) { + final String sql = """ + INSERT INTO page_relations (from_page_id, to_page_id, relation_type, value, value_type, path) + SELECT (SELECT page_id FROM pages WHERE partial_url = :fromPartialUrl), + (SELECT page_id FROM pages WHERE partial_url = :toPartialUrl), + :treeRelationType, :treeValue, :treeValueType, :treePath + """; + return new JdbcBatchItemWriterBuilder() + .dataSource(dataSource) + .sql(sql) + .itemSqlParameterSourceProvider(item -> new MapSqlParameterSource(Map.of( + "fromPartialUrl", item.fromPartialUrl(), + "toPartialUrl", item.toPartialUrl(), + "treeRelationType", item.relation().treeRelationType(), + "treeValue", item.relation().treeValue(), + "treeValueType", item.relation().treeValueType(), + "treePath", item.relation().treePath() + ))) + .build(); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java index 2c5e51597e..4a7343d832 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java @@ -23,6 +23,9 @@ public class BucketEntity { public BucketEntity() {} + public BucketEntity(Long bucketId) { + this.bucketId = bucketId; + } public BucketEntity(ViewEntity view, String bucketDescriptor) { this.view = view; diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java deleted file mode 100644 index 7765c62bf2..0000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java +++ /dev/null @@ -1,54 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity; - -import jakarta.persistence.*; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.MemberBucketEntity.BUCKETISATION; - -@Entity -@Table(name = BUCKETISATION, indexes = { - @Index(columnList = "viewName"), - @Index(columnList = "fragmentId"), - @Index(columnList = "sequenceNr") -}) -public class MemberBucketEntity { - public static final String BUCKETISATION = "fragmentation_bucketisation"; - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(columnDefinition = "serial") - private long id; - private String viewName; - private String fragmentId; - private String memberId; - private long sequenceNr; - - public MemberBucketEntity(String viewName, String fragmentId, String memberId, long sequenceNr) { - this.viewName = viewName; - this.fragmentId = fragmentId; - this.memberId = memberId; - this.sequenceNr = sequenceNr; - } - - protected MemberBucketEntity() { - - } - - public long getId() { - return id; - } - - public String getViewName() { - return viewName; - } - - public String getFragmentId() { - return fragmentId; - } - - public String getMemberId() { - return memberId; - } - - public long getSequenceNr() { - return sequenceNr; - } -} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java deleted file mode 100644 index dedbbf15c5..0000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java +++ /dev/null @@ -1,77 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity; - - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import jakarta.persistence.Embeddable; - -@Embeddable -public class TreeRelationEntity { - private String treePath; - private String treeNode; - private String treeValue; - private String treeValueType; - private String relation; - - protected TreeRelationEntity() { - } - - public TreeRelationEntity(String treePath, String treeNode, String treeValue, String treeValueType, - String relation) { - this.treePath = treePath; - this.treeNode = treeNode; - this.treeValue = treeValue; - this.treeValueType = treeValueType; - this.relation = relation; - } - - public TreeRelation toTreeRelation() { - return new TreeRelation(treePath, LdesFragmentIdentifier.fromFragmentId(treeNode), treeValue, - treeValueType, relation); - } - - public static TreeRelationEntity toEntity(TreeRelation treeRelation) { - return new TreeRelationEntity(treeRelation.treePath(), treeRelation.treeNode().asDecodedFragmentId(), - treeRelation.treeValue(), treeRelation.treeValueType(), treeRelation.relation()); - } - - public String getTreePath() { - return treePath; - } - - public void setTreePath(String treePath) { - this.treePath = treePath; - } - - public String getTreeNode() { - return treeNode; - } - - public void setTreeNode(String treeNode) { - this.treeNode = treeNode; - } - - public String getTreeValue() { - return treeValue; - } - - public void setTreeValue(String treeValue) { - this.treeValue = treeValue; - } - - public String getTreeValueType() { - return treeValueType; - } - - public void setTreeValueType(String treeValueType) { - this.treeValueType = treeValueType; - } - - public String getRelation() { - return relation; - } - - public void setRelation(String relation) { - this.relation = relation; - } -} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java index 1a514422f2..9fe3ff2221 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.BucketEntity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.projections.BucketProjection; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; @@ -12,18 +11,9 @@ private BucketMapper() { public static Bucket fromProjection(BucketProjection projection) { return new Bucket( - projection.getBucketId(), BucketDescriptor.fromString(projection.getBucketDescriptor()), ViewName.fromString(projection.getViewName()) ); } - public static Bucket fromEntity(BucketEntity entity) { - return new Bucket( - entity.getBucketId(), - BucketDescriptor.fromString(entity.getBucketDescriptor()), - ViewName.fromString(entity.getView().getComposedViewName()) - ); - } - } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java index caec723563..2ad37cf081 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java @@ -8,30 +8,29 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.jdbc.Sql; +import java.util.List; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; class BucketPostgresRepositoryTest extends PostgresBucketisationIntegrationTest { private static final ViewName VIEW_NAME = new ViewName("collection", "name"); - private static final BucketDescriptor BUCKET_DESCRIPTOR = BucketDescriptor.fromString("key=value&k=v"); @Autowired private BucketPostgresRepository bucketPostgresRepository; - @Test @Sql("./init-bucketReader.sql") void test_BucketRetrieval() { - final Bucket expectedBucket = new Bucket(BUCKET_DESCRIPTOR, VIEW_NAME); + final Bucket expectedBucket = Bucket.createRootBucketForView(VIEW_NAME); - final Optional retrievedBucket = bucketPostgresRepository.retrieveBucket(VIEW_NAME, BUCKET_DESCRIPTOR); + final Optional retrievedBucket = bucketPostgresRepository.retrieveRootBucket(VIEW_NAME); assertThat(retrievedBucket).contains(expectedBucket); } @Test void test_EmptyRetrieval() { - final Optional retrievedBucket = bucketPostgresRepository.retrieveBucket(VIEW_NAME, BUCKET_DESCRIPTOR); + final Optional retrievedBucket = bucketPostgresRepository.retrieveRootBucket(VIEW_NAME); assertThat(retrievedBucket).isEmpty(); } @@ -40,7 +39,7 @@ void test_EmptyRetrieval() { @Sql("./init-bucketWriter.sql") void test_Insertion() { final Bucket bucketToSave = new Bucket(BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME); - final Bucket expectedSavedBucket = new Bucket(1L, BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME); + final Bucket expectedSavedBucket = new Bucket(1L, BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME, List.of(), 0); final Bucket result = bucketPostgresRepository.insertBucket(bucketToSave); diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java index f858f3bb08..aed04baff8 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java @@ -4,9 +4,8 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationService; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.BucketisedMemberWriter; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates.BucketisedMemberItemWriterConfig; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; import io.cucumber.spring.CucumberContextConfiguration; import io.micrometer.observation.ObservationRegistry; import io.zonky.test.db.AutoConfigureEmbeddedDatabase; @@ -28,7 +27,7 @@ @CucumberContextConfiguration @EnableAutoConfiguration(exclude = FragmentationService.class) @DataJpaTest -@AutoConfigureEmbeddedDatabase +@AutoConfigureEmbeddedDatabase(refresh = AutoConfigureEmbeddedDatabase.RefreshMode.BEFORE_EACH_TEST_METHOD) @EnableBatchProcessing @ActiveProfiles("postgres-test") @EntityScan(basePackages = {"be.vlaanderen.informatievlaanderen.ldes.server"}) @@ -40,7 +39,7 @@ "be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view", "be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres" }) -@ContextConfiguration(classes = {BucketisedMemberWriter.class}) +@ContextConfiguration(classes = {BucketisedMemberItemWriterConfig.class}) @Import(PostgresBucketisationIntegrationTest.EventStreamControllerTestConfiguration.class) @SuppressWarnings("java:S2187") public class PostgresBucketisationIntegrationTest { @@ -55,11 +54,6 @@ public ObservationRegistry observationRegistry() { return ObservationRegistry.NOOP; } - @Bean - public MemberPropertiesRepository memberPropertiesRepository() { - return mock(MemberPropertiesRepository.class); - } - @Bean ServerMetrics serverMetrics() { return new ServerMetrics(mock(FragmentationMetricsRepository.class), mock(MemberMetricsRepository.class)); diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java new file mode 100644 index 0000000000..65d8aca472 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java @@ -0,0 +1,124 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates.TestBucketSupplier; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class BucketisationItemWriterTest { + private static final ViewName BY_TIME_VIEW_NAME = new ViewName("test", "by-time"); + private static final BucketDescriptorPair year2024Pair = new BucketDescriptorPair("year", "2024"); + private static final BucketDescriptorPair month09Pair = new BucketDescriptorPair("month", "09"); + private static final BucketDescriptorPair month08Pair = new BucketDescriptorPair("month", "08"); + private static final BucketDescriptorPair day23Pair = new BucketDescriptorPair("day", "23"); + private static final BucketDescriptorPair day26Pair = new BucketDescriptorPair("day", "26"); + + @Mock + private ItemWriter bucketItemWriter; + @Mock + private ItemWriter pageItemWriter; + @Mock + private ItemWriter bucketMemberItemWriter; + @Mock + private ItemWriter bucketRelationItemWriter; + private BucketisationItemWriter bucketisationItemWriter; + + @BeforeEach + void setUp() { + bucketisationItemWriter = new BucketisationItemWriter(bucketItemWriter, pageItemWriter, bucketMemberItemWriter, bucketRelationItemWriter); + } + + @Test + void test_ChunkOf4RootBuckets() throws Exception { + AtomicInteger memberId = new AtomicInteger(); + final Chunk chunk = Stream.of( + new BucketDescriptorPair[]{year2024Pair, month09Pair, day23Pair}, + new BucketDescriptorPair[]{year2024Pair, month09Pair, day26Pair}, + new BucketDescriptorPair[]{year2024Pair, month08Pair, day26Pair}, + new BucketDescriptorPair[]{year2024Pair, month08Pair, day23Pair} + ) + .map(pair -> new TestBucketSupplier(BY_TIME_VIEW_NAME, pair, memberId.incrementAndGet()).get()) + .collect(new ChunkCollector<>()); + + bucketisationItemWriter.write(chunk); + + verify(bucketItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(4))); + verify(pageItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(4))); + verify(bucketRelationItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(3))); + verify(bucketMemberItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(1))); + } + + @Test + void test_ChunkOf1RootBucket() throws Exception { + final int memberId = 11; + final Bucket rootBucket = new TestBucketSupplier(BY_TIME_VIEW_NAME, new BucketDescriptorPair[]{year2024Pair, month09Pair, day23Pair}, memberId).get(); + final List bucketTree = List.of( + rootBucket, + rootBucket.getChildren().getFirst(), + rootBucket.getChildren().getFirst().getChildren().getFirst(), + rootBucket.getChildren().getFirst().getChildren().getFirst().getChildren().getFirst() + ); + final List relations = new ArrayList<>(); + for (int i = 1; i < bucketTree.size(); i++) { + relations.add(new BucketRelation( + bucketTree.get(i - 1).createPartialUrl(), + bucketTree.get(i).createPartialUrl(), + TreeRelation.generic() + )); + + } + doAnswer(invocation -> { + final Chunk buckets = (Chunk) invocation.getArgument(0, Chunk.class); + List items = buckets.getItems(); + for (int i = 0; i < items.size(); i++) { + Bucket item = items.get(i); + assertThat(item.getBucketId()).isZero(); + item.setBucketId(i + 1); + } + return null; + }).when(bucketItemWriter).write(any()); + + bucketisationItemWriter.write(Chunk.of(rootBucket)); + + verify(bucketItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .asInstanceOf(InstanceOfAssertFactories.list(Bucket.class)) + .containsExactlyInAnyOrderElementsOf(bucketTree) + )); + verify(pageItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .asInstanceOf(InstanceOfAssertFactories.list(Bucket.class)) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrderElementsOf(bucketTree) + .allSatisfy(bucket -> assertThat(bucket.getBucketId()).isNotZero()) + )); + verify(bucketRelationItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .asInstanceOf(InstanceOfAssertFactories.list(BucketRelation.class)) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrderElementsOf(relations) + )); + verify(bucketMemberItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .first() + .isEqualTo(new BucketisedMember(bucketTree.size(), memberId)))); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java new file mode 100644 index 0000000000..164c9cc117 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java @@ -0,0 +1,36 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +import static org.assertj.core.api.Assertions.assertThat; + +class BucketItemWriterTest extends PostgresBucketisationIntegrationTest { + private static final ViewName VIEW_NAME = new ViewName("mobility-hindrances", "by-hour"); + @Autowired + private BucketItemWriter bucketItemWriter; + @Autowired + JdbcTemplate jdbcTemplate; + + @Test + @Sql("./init-collection-and-view.sql") + void testWriter() { + final BucketDescriptorPair[] pairs = BucketDescriptor.fromString("year=2024&month=09&day=14&hour=09").getDescriptorPairs().toArray(new BucketDescriptorPair[0]); + final Bucket rootBucket = new TestBucketSupplier(VIEW_NAME, pairs, 12).get(); + final Chunk bucketTreeChunk = new Chunk<>(rootBucket.getBucketTree()); + + bucketItemWriter.write(bucketTreeChunk); + + var count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM buckets", Integer.class); + assertThat(count).isEqualTo(5); + assertThat(bucketTreeChunk.getItems()).allSatisfy(bucket -> assertThat(bucket.getBucketId()).isNotZero()); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java new file mode 100644 index 0000000000..432d5c690a --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class BucketPageItemWriterTest extends PostgresBucketisationIntegrationTest { + private static final ViewName VIEW_NAME = new ViewName("mobility-hindrances", "by-hour"); + @Autowired + private ItemWriter pageItemWriter; + @Autowired + JdbcTemplate jdbcTemplate; + + @Test + @Sql({"./init-collection-and-view.sql", "./init-writer-test.sql"}) + void testWriter() throws Exception { + final BucketDescriptorPair[] pairs = BucketDescriptor.fromString("year=2023&month=06").getDescriptorPairs().toArray(new BucketDescriptorPair[0]); + final List bucketTree = new TestBucketSupplier(VIEW_NAME, pairs, 12, true).getBucketTree(); + final Chunk bucketTreeChunk = new Chunk<>(bucketTree); + + pageItemWriter.write(bucketTreeChunk); + + var count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM pages", Integer.class); + assertThat(count).isEqualTo(3); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberWriterTest.java similarity index 65% rename from ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriterTest.java rename to ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberWriterTest.java index b3765e3fe1..b658e77865 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriterTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberWriterTest.java @@ -1,41 +1,38 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; import org.junit.jupiter.api.Test; import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.context.jdbc.Sql; import javax.sql.DataSource; -import java.util.List; import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; class BucketisedMemberWriterTest extends PostgresBucketisationIntegrationTest { @Autowired - BucketisedMemberWriter writer; + ItemWriter writer; @Autowired DataSource dataSource; @Test - @Sql("./init-writer-test.sql") + @Sql({"./init-collection-and-view.sql", "./init-writer-test.sql"}) void testWriter() throws Exception { - List bucketisedMembers = initBucketisedMembers(); - writer.write(Chunk.of(List.of(bucketisedMembers.get(0), bucketisedMembers.get(1)), - List.of(bucketisedMembers.get(2)))); + final long bucketId = 3; + final Chunk members = IntStream.range(1, 4) + .mapToObj(memberId -> new BucketisedMember(bucketId, memberId)) + .collect(new ChunkCollector<>()); - var count = new JdbcTemplate(dataSource).queryForObject("SELECT COUNT(*) FROM page_members", Integer.class); + writer.write(members); + var count = new JdbcTemplate(dataSource).queryForObject("SELECT COUNT(*) FROM page_members", Integer.class); assertThat(count).isEqualTo(3); } - - private static List initBucketisedMembers() { - return IntStream.range(1, 4) - .mapToObj(id -> new BucketisedMember(1, id)) - .toList(); - } } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java new file mode 100644 index 0000000000..9502c8023f --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class PageRelationItemWriterTest extends PostgresBucketisationIntegrationTest { + private static final ViewName VIEW_NAME = new ViewName("mobility-hindrances", "by-hour"); + + @Autowired + private ItemWriter relationItemWriter; + @Autowired + private JdbcTemplate jdbcTemplate; + + @Test + @Sql({"./init-collection-and-view.sql", "./init-writer-test.sql", "./insert-pages.sql"}) + void testWriter() throws Exception { + final BucketDescriptorPair[] pairs = BucketDescriptor.fromString("year=2023&month=06").getDescriptorPairs().toArray(new BucketDescriptorPair[0]); + final Chunk chunk = new TestBucketSupplier(VIEW_NAME, pairs, 12, true).getBucketTree().stream() + .flatMap(bucket -> bucket.getChildRelations().stream()) + .collect(new ChunkCollector<>()); + chunk.add(new BucketRelation("/mobility-hindrances/by-hour?year=2023", "/mobility-hindrances/by-hour?year=2023&month=07", TreeRelation.generic())); + + relationItemWriter.write(chunk); + + var count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM page_relations", Integer.class); + assertThat(count).isEqualTo(3); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java new file mode 100644 index 0000000000..1d18161e30 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java @@ -0,0 +1,54 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; + +import java.util.List; +import java.util.function.Supplier; + +public class TestBucketSupplier implements Supplier { + private final ViewName viewName; + private final BucketDescriptorPair[] bucketDescriptorPairs; + private final long memberId; + private final boolean idGenerationEnabled; + private int currentId = 0; + + public TestBucketSupplier(ViewName viewName, BucketDescriptorPair[] bucketDescriptorPairs, long memberId, boolean idGenerationEnabled) { + this.viewName = viewName; + this.bucketDescriptorPairs = bucketDescriptorPairs; + this.memberId = memberId; + this.idGenerationEnabled = idGenerationEnabled; + } + + public TestBucketSupplier(ViewName viewName, BucketDescriptorPair[] bucketDescriptorPairs, long memberId) { + this(viewName, bucketDescriptorPairs, memberId, false); + } + + @Override + public Bucket get() { + Bucket rootBucket = Bucket.createRootBucketForView(viewName); + assignBucketIdIfNecessary(rootBucket); + Bucket nextParent = rootBucket; + for (final BucketDescriptorPair bucketDescriptorPair : bucketDescriptorPairs) { + nextParent = createChildBucket(nextParent, bucketDescriptorPair); + assignBucketIdIfNecessary(nextParent); + } + nextParent.assignMember(memberId); + return rootBucket; + } + + public List getBucketTree() { + return get().getBucketTree(); + } + + private Bucket createChildBucket(Bucket parentBucket, BucketDescriptorPair childDescriptorPair) { + return parentBucket.addChildBucket(parentBucket.createChild(childDescriptorPair).withGenericRelation()); + } + + private void assignBucketIdIfNecessary(Bucket bucket) { + if (idGenerationEnabled) { + bucket.setBucketId(++currentId); + } + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql new file mode 100644 index 0000000000..d23e72763f --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql @@ -0,0 +1,14 @@ +INSERT INTO collections (collection_id, name, timestamp_path, version_of_path, create_versions, is_closed) +VALUES (1, 'mobility-hindrances', 'http://www.w3.org/ns/prov#generatedAtTime', 'http://purl.org/dc/terms/isVersionOf', + false, false); + +INSERT INTO views (view_id, collection_id, name, fragmentations, retention_policies, page_size) +VALUES (1, 1, 'by-hour', '[ + { + "name": "HierarchicalTimeBasedFragmentation", + "config": { + "maxGranularity": "hour", + "fragmentationPath": "http://www.w3.org/ns/prov#generatedAtTime" + } + } +]', '', 250); diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/init-writer-test.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql similarity index 99% rename from ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/init-writer-test.sql rename to ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql index 7765171988..56591c9cbe 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/init-writer-test.sql +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql @@ -1,18 +1,3 @@ -INSERT INTO collections (collection_id, name, timestamp_path, version_of_path, create_versions, is_closed) -VALUES (1, 'mobility-hindrances', 'http://www.w3.org/ns/prov#generatedAtTime', 'http://purl.org/dc/terms/isVersionOf', - false, false); - -INSERT INTO views (view_id, collection_id, name, fragmentations, retention_policies, page_size) -VALUES (1, 1, 'by-hour', '[ - { - "name": "HierarchicalTimeBasedFragmentation", - "config": { - "maxGranularity": "hour", - "fragmentationPath": "http://www.w3.org/ns/prov#generatedAtTime" - } - } -]', '', 250); - INSERT INTO members (member_id, subject, old_id, collection_id, is_in_event_source, member_model, timestamp, transaction_id, version_of) VALUES (1, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600000', @@ -40,4 +25,6 @@ VALUES (3, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindran '2024-06-27 08:47:04.004000', '9f3a0219-e12d-4891-859f-6c9c5f967d48', 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400'); -INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (1, 'year=2023&month=06', 1); \ No newline at end of file +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (1, '', 1); +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (2, 'year=2023', 1); +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (3, 'year=2023&month=06', 1); \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql new file mode 100644 index 0000000000..1ce826fed1 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql @@ -0,0 +1,7 @@ +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (4, 'year=2023&month=07', 1); + +INSERT INTO pages (page_id, bucket_id, partial_url) +VALUES (1, 1, '/mobility-hindrances/by-hour'), + (2, 2, '/mobility-hindrances/by-hour?year=2023'), + (3, 3, '/mobility-hindrances/by-hour?year=2023&month=06'), + (4, 4, '/mobility-hindrances/by-hour?year=2023&month=07') \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql index c0d9658975..14e5a4423b 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql @@ -4,4 +4,5 @@ VALUES (1, 'collection', 'http://purl.org/dc/terms/created', 'http://purl.org/dc INSERT INTO views VALUES (1, 1, 'name', '[]', '', 150); -INSERT INTO buckets VALUES (1, 'key=value&k=v', 1) \ No newline at end of file +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (1, 'key=value&k=v', 1); +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (2, '', 1) \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml index 2e0949d4f9..234333ebcf 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0 + 3.4.0-SNAPSHOT postgres-ingest-repository diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java index e6d7b940e2..a3fe127185 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java @@ -15,9 +15,11 @@ import java.util.HashMap; import java.util.Map; +import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch.BucketJobDefinitions.CHUNK_SIZE; + @Configuration public class MemberItemReader { - private static final int PAGE_SIZE = 500; + private static final int PAGE_SIZE = CHUNK_SIZE; @Bean @StepScope @@ -30,7 +32,7 @@ public JdbcPagingItemReader memberReader(@Value("#{jobParam .queryProvider(memberQuery()) .parameterValues(jobParameters) .pageSize(PAGE_SIZE) - .maxItemCount(20 * PAGE_SIZE) + .maxItemCount(40 * PAGE_SIZE) .build(); } @@ -38,19 +40,25 @@ private PostgresPagingQueryProvider memberQuery() { Map sortKeys = new HashMap<>(); sortKeys.put("member_id", Order.ASCENDING); PostgresPagingQueryProvider queryProvider = new PostgresPagingQueryProvider(); - queryProvider.setSelectClause("m.member_id, m.subject, m.version_of, m.timestamp, c.name, c.version_of_path, c.timestamp_path, c.create_versions, m.member_model"); + queryProvider.setSelectClause(""" + m.member_id, m.subject, m.version_of, m.timestamp, + c.name, c.version_of_path, c.timestamp_path, c.create_versions, + m.member_model + """); queryProvider.setFromClause(""" - collections c - join views v on v.collection_id = c.collection_id - join bucket_stats bs on bs.collection_id = c.collection_id and bs.view_id = v.view_id - join members m on m.collection_id = c.collection_id + collections c + join members m on m.collection_id = c.collection_id """); queryProvider.setWhereClause(""" - m.member_id > bs.last - AND v.name = :viewName AND c.name = :collectionName + c.name = :collectionName and + m.member_id > ( + COALESCE( + (SELECT max(pm.member_id) FROM page_members pm + WHERE pm.view_id IN (select v.view_id from views v where v.name = :viewName)), + (0)::bigint) + ) """); queryProvider.setSortKeys(sortKeys); return queryProvider; } - } diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java index 86a78274ed..038962c80c 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java @@ -51,6 +51,10 @@ public MemberEntity(String subject, EventStreamEntity collection, String version this.model = model; } + public MemberEntity(long id) { + this.id = id; + } + protected MemberEntity() { } diff --git a/ldes-server-infra-postgres/postgres-liquibase/pom.xml b/ldes-server-infra-postgres/postgres-liquibase/pom.xml index 74b04081cb..f2c0848f62 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/pom.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0 + 3.4.0-SNAPSHOT postgres-liquibase diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml new file mode 100644 index 0000000000..f7c5a9b582 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql new file mode 100644 index 0000000000..22dbd4703f --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql @@ -0,0 +1,65 @@ +-- create view statistics table +CREATE TABLE view_stats +( + view_id bigint NOT NULL, + bucketized_count bigint NOT NULL default 0, + paginated_count bigint NOT NULL default 0 +); +ALTER TABLE "view_stats" ADD FOREIGN KEY ("view_id") REFERENCES "views" ("view_id") ON DELETE CASCADE; + +-- create function to insert a stats row for the new view +create function on_view_inserted() returns trigger language plpgsql as $$ +begin + insert into view_stats(view_id) values (NEW.view_id); + return null; +end +$$; + +-- add stats row after view inserted +create trigger views_ai after insert on views +for each row execute procedure on_view_inserted(); + +-- create function to delete stats row for a view +create function on_view_deleted() returns trigger language plpgsql as $$ +begin + delete from view_stats where view_id = OLD.view_id; + return null; +end +$$; + +-- remove stats row after view deleted +create trigger views_ad after delete on views +for each row execute procedure on_view_deleted(); + +-- initialize current counts +update view_stats vs +set + bucketized_count = (select count(*) from page_members pm WHERE pm.view_id = vs.view_id), + paginated_count = (select count(*) from page_members pm WHERE pm.view_id = vs.view_id and pm.page_id is not null) +; + +-- create function to increase view stats count when member bucketized +create function on_page_member_inserted() returns trigger language plpgsql as $$ +begin + update view_stats set bucketized_count = bucketized_count + 1 where view_id = NEW.view_id; + return null; +end +$$; + +-- update stats row after page_member inserted +create trigger page_member_ai after insert on page_members +for each row execute procedure on_page_member_inserted(); + +-- create function to increase view stats count when member paginated +create function on_page_member_updating() returns trigger language plpgsql as $$ +begin + if (OLD.page_id is null and NEW.page_id is not null) then + update view_stats set paginated_count = paginated_count + 1 where view_id = NEW.view_id; + end if; + return NEW; +end +$$; + +-- update stats row before page_member updating +create trigger page_member_bu before update on page_members +for each row execute procedure on_page_member_updating(); diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql new file mode 100644 index 0000000000..d0306ce3b3 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql @@ -0,0 +1,5 @@ +create or replace view bucket_last_page as +select p.bucket_id, max(p.page_id) as last_page_id +from pages p +group by p.bucket_id +; \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql new file mode 100644 index 0000000000..0092221421 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql @@ -0,0 +1,2 @@ +DROP TRIGGER insert_page_on_bucket_insertion ON buckets; +DROP FUNCTION on_bucket_insertion; \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql new file mode 100644 index 0000000000..4fb4b624a9 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql @@ -0,0 +1,3 @@ +UPDATE page_members pm +SET view_id = (select b.view_id from buckets b where b.bucket_id = pm.bucket_id) +; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml new file mode 100644 index 0000000000..afb75413f4 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql new file mode 100644 index 0000000000..4089f24342 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW bucket_stats as +select c.collection_id, v.view_id, + coalesce((select max(pm.member_id) from page_members pm where v.view_id = pm.view_id),0) as last +from collections c +inner join views v on c.collection_id = v.collection_id +group by c.collection_id, v.view_id; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql new file mode 100644 index 0000000000..ca25b91f0a --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE VIEW "needs_bucketization" AS +SELECT ms.collection_id, ms.view_id, (pm.member_id is null) AS should_bucketize +FROM member_stats ms +left outer join page_members pm on pm.view_id = ms.view_id and pm.member_id = ms.last +group by ms.collection_id, ms.view_id, pm.member_id diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql new file mode 100644 index 0000000000..79990504bf --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW "needs_pagination" AS +SELECT c.collection_id, v.view_id, + (EXISTS (SELECT * FROM page_members pm WHERE pm.page_id IS NULL AND pm.view_id = v.view_id)) AS should_paginate +FROM collections c +JOIN views v ON c.collection_id = v.collection_id +GROUP BY c.collection_id, v.view_id; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index f054e4f7ef..3061e20c4c 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -7,5 +7,6 @@ + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml index 6e7b900080..4774ad5898 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0 + 3.4.0-SNAPSHOT postgres-pagination-repository @@ -44,6 +44,12 @@ spring-batch-test test + + be.vlaanderen.informatievlaanderen.vsds + postgres-liquibase + ${project.version} + test + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java index fcbe111d7c..9037e48cfb 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java @@ -1,6 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageMemberEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; import org.springframework.data.jpa.repository.Modifying; @@ -10,13 +12,13 @@ import java.util.List; @Repository -public class PageMemberPostgresRepository implements PageMemberRepository { +public class PageMemberPostgresRepository implements PageMemberRepository, be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository { private final PageMemberEntityRepository entityRepository; - public PageMemberPostgresRepository(PageMemberEntityRepository entityRepository) { + public PageMemberPostgresRepository(PageMemberEntityRepository entityRepository) { this.entityRepository = entityRepository; - } + } @Override @Transactional @@ -30,4 +32,14 @@ public void setPageMembersToNewPage(long newPageId, List pageIds) { public void deleteByViewNameAndMembersIds(ViewName viewName, List memberIds) { entityRepository.deleteAllByBucket_View_EventStream_NameAndBucket_View_NameAndMember_IdIn(viewName.getCollectionName(), viewName.getViewName(), memberIds); } + + @Override + public List getUnpaginatedMembersForBucket(long bucketId) { + return entityRepository.findByBucketIdAndPageIdIsNullOrderByMemberId(bucketId); + } + + @Override + public void assignMembersToPage(Page openPage, List pageMembers) { + entityRepository.updatePageForMembers(new PageEntity(openPage.getId()), openPage.getBucketId(), pageMembers); + } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java index d625537e2a..56a2cd20b8 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java @@ -4,7 +4,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch.PaginationRowMapper; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; @@ -12,31 +14,33 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.Objects; import java.util.stream.Stream; @Repository public class PagePostgresRepository implements PageRepository { private final JdbcTemplate jdbcTemplate; private final PageEntityRepository pageEntityRepository; + private final PageRelationRepository pageRelationRepository; - public PagePostgresRepository(JdbcTemplate jdbcTemplate, PageEntityRepository pageEntityRepository) { + public PagePostgresRepository(JdbcTemplate jdbcTemplate, PageEntityRepository pageEntityRepository, PageRelationRepository pageRelationRepository) { this.jdbcTemplate = jdbcTemplate; this.pageEntityRepository = pageEntityRepository; + this.pageRelationRepository = pageRelationRepository; } @Override @Transactional(readOnly = true) public Page getOpenPage(long bucketId) { String sql = """ - select DISTINCT p.page_id, p.bucket_id, p.partial_url, v.page_size, COUNT(member_id) AS assigned_members - FROM pages p - LEFT JOIN page_members m ON p.page_id = m.page_id - JOIN buckets b ON p.bucket_id = b.bucket_id - JOIN views v ON v.view_id = b.view_id - where b.bucket_id = ? AND p.page_id NOT IN (SELECT from_page_id FROM page_relations) - group by p.page_id, v.page_size - order by page_id + select p.page_id, p.bucket_id, p.partial_url, v.page_size, COUNT(member_id) AS assigned_members + from pages p + left join page_members pm on pm.page_id = p.page_id + JOIN buckets b ON p.bucket_id = b.bucket_id + JOIN views v ON v.view_id = b.view_id + join bucket_last_page blp on blp.bucket_id = b.bucket_id AND blp.last_page_id = p.page_id + where b.bucket_id = ? + group by p.page_id, v.page_size + order by page_id """; return jdbcTemplate.query(sql, new PaginationRowMapper(), bucketId) .stream() @@ -46,27 +50,15 @@ select DISTINCT p.page_id, p.bucket_id, p.partial_url, v.page_size, COUNT(member @Override @Transactional - public int createPage(Long bucketId, String partialUrl) { - String sql = """ - INSERT INTO pages (bucket_id, expiration, partial_url) - VALUES (?, NULL, ?) - ON CONFLICT (partial_url) DO UPDATE SET bucket_id = pages.bucket_id - RETURNING page_id; - """; + public Page createNextPage(Page parentPage) { + String partialUrl = parentPage.createChildPartialUrl().asString(); + PageEntity newPage = new PageEntity(parentPage.getBucketId(), partialUrl); - return Objects.requireNonNull(jdbcTemplate.queryForObject(sql, Long.class, bucketId, partialUrl)).intValue(); - } + pageEntityRepository.save(newPage); + pageRelationRepository.insertGenericBucketRelation(parentPage.getId(), newPage.getId()); + pageEntityRepository.setPageImmutable(parentPage.getId()); - @Override - @Transactional - public void setPageImmutable(long pageId) { - pageEntityRepository.setPageImmutable(pageId); - } - - @Override - @Transactional - public void setChildrenImmutableByBucketId(long bucketId) { - pageEntityRepository.setAllChildrenImmutableByBucketId(bucketId); + return new Page(newPage.getId(), parentPage.getBucketId(), partialUrl, parentPage.getPageSize()); } @Override diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java index 7129565ed7..f894a98293 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java @@ -1,8 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.RelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; @@ -11,34 +10,21 @@ @Repository public class PageRelationPostgresRepository implements PageRelationRepository { - private final RelationEntityRepository relationEntityRepository; + private final PageRelationEntityRepository pageRelationEntityRepository; - public PageRelationPostgresRepository(RelationEntityRepository relationEntityRepository) { - this.relationEntityRepository = relationEntityRepository; + public PageRelationPostgresRepository(PageRelationEntityRepository pageRelationEntityRepository) { + this.pageRelationEntityRepository = pageRelationEntityRepository; } @Override @Transactional public void insertGenericBucketRelation(long fromPageId, long toPageId) { - relationEntityRepository.insertRelation(fromPageId, toPageId, RdfConstants.GENERIC_TREE_RELATION); - } - - @Override - @Transactional - public void insertBucketRelation(BucketRelation bucketRelation) { - relationEntityRepository.insertRelation( - bucketRelation.fromBucket().createPartialUrl(), - bucketRelation.toBucket().createPartialUrl(), - bucketRelation.treeRelationType(), - bucketRelation.treeValue(), - bucketRelation.treeValueType(), - bucketRelation.treePath() - ); + pageRelationEntityRepository.insertRelation(fromPageId, toPageId, RdfConstants.GENERIC_TREE_RELATION); } @Override @Transactional public void updateCompactionBucketRelations(List compactedPageIds, long targetId) { - relationEntityRepository.updateToPageRelations(compactedPageIds, targetId); + pageRelationEntityRepository.updateToPageRelations(compactedPageIds, targetId); } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java index 89e41ac970..e3fb6d592c 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java @@ -3,7 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.EventStream; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; @@ -13,7 +13,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageMemberEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.RelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Repository; @@ -25,13 +25,13 @@ @Repository public class TreeNodePostgresRepository implements TreeNodeRepository { private final PageEntityRepository pageEntityRepository; - private final RelationEntityRepository relationEntityRepository; + private final PageRelationEntityRepository pageRelationEntityRepository; private final PageMemberEntityRepository pageMemberEntityRepository; private final Map versionObjectCreatorMap = new HashMap<>(); - public TreeNodePostgresRepository(PageEntityRepository pageEntityRepository, RelationEntityRepository relationEntityRepository, PageMemberEntityRepository pageMemberEntityRepository) { + public TreeNodePostgresRepository(PageEntityRepository pageEntityRepository, PageRelationEntityRepository pageRelationEntityRepository, PageMemberEntityRepository pageMemberEntityRepository) { this.pageEntityRepository = pageEntityRepository; - this.relationEntityRepository = relationEntityRepository; + this.pageRelationEntityRepository = pageRelationEntityRepository; this.pageMemberEntityRepository = pageMemberEntityRepository; } @@ -40,7 +40,7 @@ public Optional findByFragmentIdentifier(LdesFragmentIdentifier fragme return pageEntityRepository .findTreeNodeByPartialUrl(fragmentIdentifier.asDecodedFragmentId()) .map(page -> { - final List relations = relationEntityRepository.findDistinctByFromPageId(page.getId()); + final List relations = pageRelationEntityRepository.findDistinctByFromPageId(page.getId()); var versionObjectCreator = versionObjectCreatorMap.get(page.getCollectionName()); @@ -60,7 +60,7 @@ public Optional findTreeNodeWithoutMembers(LdesFragmentIdentifier frag return pageEntityRepository .findTreeNodeByPartialUrl(fragmentIdentifier.asDecodedFragmentId()) .map(page -> { - final List relations = relationEntityRepository.findDistinctByFromPageId(page.getId()); + final List relations = pageRelationEntityRepository.findDistinctByFromPageId(page.getId()); return TreeNodeMapper.fromProjection(page, relations, List.of()); }); } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java index 40bdd806ce..a31f7f2d08 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java @@ -17,8 +17,7 @@ public class BucketPartitioner implements Partitioner{ static final String SQL = """ SELECT pm.bucket_id FROM page_members pm - JOIN buckets b on b.bucket_id = pm.bucket_id - JOIN views v on v.view_id = b.view_id + JOIN views v on v.view_id = pm.view_id JOIN collections c on c.collection_id = v.collection_id WHERE c.name = ? AND v.name = ? AND page_id IS NULL GROUP BY pm.bucket_id diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java deleted file mode 100644 index ee91c6c072..0000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java +++ /dev/null @@ -1,59 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.database.JdbcBatchItemWriter; -import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import javax.sql.DataSource; -import java.util.Collection; -import java.util.List; - -@Component -public class PageAssignmentsWriter implements ItemWriter> { - private final JdbcBatchItemWriter delegateWriter; - - public PageAssignmentsWriter(JdbcBatchItemWriter batchItemWriter) { - this.delegateWriter = batchItemWriter; - } - - @Override - - public void write(Chunk> chunk) throws Exception { - var items = chunk.getItems() - .stream() - .flatMap(Collection::stream) - .toList(); - - if (!items.isEmpty()) { - delegateWriter.write(new Chunk<>(items)); - } - } - - @Configuration - public static class BatchPageAssignmentWriterConfig { - private static final String SQL = """ - UPDATE page_members - SET page_id = ? - WHERE bucket_id = ? AND member_id = ? - """; - - @Bean - JdbcBatchItemWriter batchPageAssignmentWriter(DataSource dataSource) { - return new JdbcBatchItemWriterBuilder() - .dataSource(dataSource) - .sql(SQL) - .itemPreparedStatementSetter((item, ps) -> { - ps.setLong(1, item.pageId()); - ps.setLong(2, item.bucketId()); - ps.setLong(3, item.memberId()); - }) - .build(); - } - } - -} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java deleted file mode 100644 index 46b0f23d8c..0000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java +++ /dev/null @@ -1,80 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.batch.item.ItemStreamReader; -import org.springframework.batch.item.database.JdbcCursorItemReader; -import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; - -import javax.sql.DataSource; -import java.util.ArrayList; -import java.util.List; - -import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch.PaginationJobDefinitions.CHUNK_SIZE; - - -@Component -@StepScope -public class UnpagedReader implements ItemStreamReader> { - private static final String SQL = """ - SELECT member_id, bucket_id - FROM page_members - WHERE bucket_id = ? - AND page_id IS NULL - ORDER BY member_id - """; - private final ItemStreamReader delegate; - - public UnpagedReader(ItemStreamReader delegate) { - this.delegate = delegate; - } - - @Override - public List read() throws Exception { - List items = new ArrayList<>(CHUNK_SIZE); - - for (int i = 0; i < CHUNK_SIZE; i++) { - UnpagedMember item = delegate.read(); - if (item == null) { - // If there are no more items, return what we have (even if it's less than the bundle size) - return items.isEmpty() ? null : items; - } - items.add(item); - } - - return items; - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - delegate.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - delegate.update(executionContext); - } - - @Override - public void close() throws ItemStreamException { - delegate.close(); - } - - @Bean - @StepScope - JdbcCursorItemReader delegateReader(DataSource dataSource, - @Value("#{stepExecutionContext['bucketId']}") Long bucketId) { - return new JdbcCursorItemReaderBuilder() - .name("unpagedReader") - .dataSource(dataSource) - .sql(SQL) - .preparedStatementSetter(ps -> ps.setLong(1, bucketId)) - .rowMapper((rs, rowNum) -> new UnpagedMember(rs.getLong(1), rs.getLong(2))) - .build(); - } -} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java index 104c77a579..7bd32d38c5 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java @@ -31,16 +31,19 @@ public class PageEntity { private String partialUrl; @OneToMany(mappedBy = "fromPage") - private List relations; + private List relations; public PageEntity() { } - public PageEntity(BucketEntity bucket, String partialUrl, List relations) { - this.bucket = bucket; - this.immutable = true; + public PageEntity(Long pageId) { + this.id = pageId; + } + + public PageEntity(Long bucketId, String partialUrl) { + this.bucket = new BucketEntity(bucketId); + this.immutable = false; this.partialUrl = partialUrl; - this.relations = relations; } public Long getId() { @@ -67,11 +70,15 @@ public String getPartialUrl() { return partialUrl; } - public List getRelations() { + public List getRelations() { return relations; } public boolean isView() { return bucket.getView().getComposedViewName().equals("/%s".formatted(partialUrl)); } + + public void setId(Long id) { + this.id = id; + } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java index fd5fdf72f3..6612d9a235 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java @@ -28,4 +28,24 @@ public class PageMemberEntity { @OnDelete(action = OnDeleteAction.CASCADE) @JoinColumn(name = "page_id", columnDefinition = "BIGINT") private PageEntity page; + + public void setPage(PageEntity page) { + this.page = page; + } + + public void setMember(Long memberId) { + this.member = new MemberEntity(memberId); + } + + public MemberEntity getMember() { + return member; + } + + public BucketEntity getBucket() { + return bucket; + } + + public PageEntity getPage() { + return page; + } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java index 6f1e84351b..b20c6c4bfa 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java @@ -13,6 +13,14 @@ public class PageMemberId implements Serializable { @Column(name = "bucket_id", nullable = false, columnDefinition = "BIGINT") private Long bucketId; + public PageMemberId() { + } + + public PageMemberId(Long memberId, Long bucketId) { + this.memberId = memberId; + this.bucketId = bucketId; + } + @Override public final boolean equals(Object o) { if (this == o) return true; diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/RelationEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageRelationEntity.java similarity index 65% rename from ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/RelationEntity.java rename to ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageRelationEntity.java index 2e569cf011..28053b7e7b 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/RelationEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageRelationEntity.java @@ -6,7 +6,7 @@ @Entity @Table(name = "page_relations") -public class RelationEntity { +public class PageRelationEntity { @EmbeddedId private RelationId relationId; @@ -34,35 +34,13 @@ public class RelationEntity { @Column(name = "path", columnDefinition = "VARCHAR(255)") private String treePath; - - public RelationEntity() { - } - - public RelationEntity(PageEntity fromPage, PageEntity toPage, String treeRelationType, String treeValue, String treeValueType, String treePath) { - this.fromPage = fromPage; - this.toPage = toPage; - this.treeRelationType = treeRelationType; - this.treeValue = treeValue; - this.treeValueType = treeValueType; - this.treePath = treePath; - } - - public RelationEntity(RelationId relationId, String treeRelationType, String treeValue, String treeValueType, String treePath) { - this.relationId = relationId; - this.treeRelationType = treeRelationType; - this.treeValue = treeValue; - this.treeValueType = treeValueType; - this.treePath = treePath; + protected PageRelationEntity() { } public RelationId getRelationId() { return relationId; } - public PageEntity getFromPage() { - return fromPage; - } - public PageEntity getToPage() { return toPage; } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java index a6ca3a68a9..3cec2777cd 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java @@ -1,23 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.mapper; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.RelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; public class TreeRelationMapper { private TreeRelationMapper() {} - public static TreeRelation fromRelationEntity(RelationEntity relationEntity) { - return new TreeRelation( - relationEntity.getTreePath(), - LdesFragmentIdentifier.fromFragmentId(relationEntity.getToPage().getPartialUrl()), - relationEntity.getTreeValue(), - relationEntity.getTreeValueType(), - relationEntity.getTreeRelationType() - ); - } - public static TreeRelation fromProjection(TreeRelationProjection projection) { return new TreeRelation( projection.getTreePath(), diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java index 54558b5551..289e4a77f3 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java @@ -21,18 +21,6 @@ public interface PageEntityRepository extends JpaRepository { @Query(value = "UPDATE pages SET immutable = true WHERE page_id = ?", nativeQuery = true) void setPageImmutable(long pageId); - @Modifying - @Query(value = """ - update pages set immutable = true - where page_id in ( - select distinct r.to_page_id from pages p - inner join buckets b on b.bucket_id = p.bucket_id - inner join page_relations r on r.from_page_id = p.page_id - where b.bucket_id = :bucketId - ) - """, nativeQuery = true) - void setAllChildrenImmutableByBucketId(long bucketId); - @Modifying @Query(value = """ UPDATE pages @@ -47,7 +35,7 @@ JOIN views USING (view_id) @Query(value = "SELECT p.id as fragmentId, COUNT(*) AS size, r.toPage.id AS toPage, p.immutable AS immutable, " + "p.expiration AS expiration, " + "p.bucket.bucketId AS bucketId, p.partialUrl AS partialUrl " + - "FROM PageEntity p JOIN BucketEntity b ON p.bucket = b JOIN ViewEntity v ON b.view = v JOIN RelationEntity r ON p = r.fromPage " + + "FROM PageEntity p JOIN BucketEntity b ON p.bucket = b JOIN ViewEntity v ON b.view = v JOIN PageRelationEntity r ON p = r.fromPage " + "WHERE v.eventStream.name = :collectionName AND v.name = :viewName " + "GROUP BY p.id, r.toPage.id " + "HAVING COUNT(*) < :capacityPerPage") diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java index 860a3ea641..19214a1dab 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java @@ -1,12 +1,14 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.projection.TreeMemberProjection; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageMemberEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageMemberId; import jakarta.persistence.Tuple; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import java.util.List; @@ -14,31 +16,38 @@ public interface PageMemberEntityRepository extends JpaRepository findAllMembersByPageId(long pageId); - @Modifying - @Query("UPDATE PageMemberEntity m SET m.page = (SELECT p FROM PageEntity p WHERE p.id = :newPageId) WHERE m.page.id IN (:pageIds)") - void setPageMembersToNewPage(long newPageId, List pageIds); + @Modifying + @Query("UPDATE PageMemberEntity m SET m.page = (SELECT p FROM PageEntity p WHERE p.id = :newPageId) WHERE m.page.id IN (:pageIds)") + void setPageMembersToNewPage(long newPageId, List pageIds); @Query(value = """ - select v.name, count(*) - from page_members - JOIN buckets b on b.bucket_id = page_members.bucket_id - JOIN views v on v.view_id = b.view_id - JOIN collections c on c.collection_id = v.collection_id - WHERE c.name = :collectionName - group by v.name - """, nativeQuery = true) + select v.name, vs.bucketized_count as count + from view_stats vs + JOIN views v on v.view_id = vs.view_id + JOIN collections c on c.collection_id = v.collection_id + WHERE c.name = :collectionName + """, nativeQuery = true) List getBucketisedMemberCounts(String collectionName); @Query(value = """ - select v.name, count(*) - from page_members - JOIN buckets b on b.bucket_id = page_members.bucket_id - JOIN views v on v.view_id = b.view_id - JOIN collections c on c.collection_id = v.collection_id - WHERE page_id IS NOT NULL AND c.name = :collectionName - group by v.name - """, nativeQuery = true) + select v.name, vs.paginated_count as count + from view_stats vs + JOIN views v on v.view_id = vs.view_id + JOIN collections c on c.collection_id = v.collection_id + WHERE c.name = :collectionName + """, nativeQuery = true) List getPaginatedMemberCounts(String collectionName); + @Modifying + @Query(""" + UPDATE PageMemberEntity p SET p.page = :page WHERE p.pageMemberId.bucketId = :bucketId + AND p.pageMemberId.memberId IN :memberIds + """) + void updatePageForMembers(@Param("page") PageEntity page, @Param("bucketId") Long bucketId, @Param("memberIds") List memberIds); + + @Query("SELECT pm.pageMemberId.memberId FROM PageMemberEntity pm WHERE pm.pageMemberId.bucketId = :bucketId AND " + + "pm.page IS NULL ORDER BY pm.pageMemberId.memberId") + List findByBucketIdAndPageIdIsNullOrderByMemberId(long bucketId); + void deleteAllByBucket_View_EventStream_NameAndBucket_View_NameAndMember_IdIn(String collectionName, String viewName, List memberIds); } \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/RelationEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageRelationEntityRepository.java similarity index 85% rename from ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/RelationEntityRepository.java rename to ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageRelationEntityRepository.java index 02f3293fd6..7f4bf5f80e 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/RelationEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageRelationEntityRepository.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.RelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.RelationId; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; import org.springframework.data.jpa.repository.JpaRepository; @@ -9,7 +9,7 @@ import java.util.List; -public interface RelationEntityRepository extends JpaRepository { +public interface PageRelationEntityRepository extends JpaRepository { @Modifying @Query(value = """ @@ -29,7 +29,7 @@ INSERT INTO page_relations (from_page_id, to_page_id, relation_type, value, valu List findDistinctByFromPageId(long pageId); @Modifying - @Query("UPDATE RelationEntity r SET r.toPage = ( SELECT p FROM PageEntity p WHERE p.id = :targetId ) " + + @Query("UPDATE PageRelationEntity r SET r.toPage = ( SELECT p FROM PageEntity p WHERE p.id = :targetId ) " + "WHERE r.toPage.id IN :ids OR r.fromPage.id IN : ids") void updateToPageRelations(List ids, long targetId); } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java new file mode 100644 index 0000000000..395bb21d35 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java @@ -0,0 +1,119 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.BucketEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import io.cucumber.java.Before; +import io.cucumber.java.en.And; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.stream.LongStream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PaginationSteps extends PostgresPaginationIntegrationTest { + private List memberIds; + private int pageCount; + private Long bucketId; + private Integer viewId; + private Page openPage; + + @Before + public void setUp() { + pageCount = pageEntityRepository.findAll().size(); + final BucketEntity bucket = bucketEntityRepository.findAll().getFirst(); + bucketId = bucket.getBucketId(); + viewId = bucket.getView().getId(); + openPage = pageRepository.getOpenPage(bucketId); + } + + @Given("I have {int} unpaged members in one bucket") + public void iHaveUnpagedMembersForBucket(int memberCount) { + memberIds = LongStream.rangeClosed(1, memberCount).boxed().toList(); + + saveMembers(memberIds); + saveMemberBuckets(memberIds); + } + + @When("I assign the members to the page") + public void iAssignTheMembersToThePage() { + pageMemberRepository.assignMembersToPage(openPage, memberIds); + } + + @Then("I expect a new page is created") + public void iExpectANewPageIsCreated() { + var currentPageCount = pageEntityRepository.count(); + assertThat(currentPageCount).isGreaterThan(pageCount); + } + + @Then("the open page has space for {int} more members") + public void theOpenPageContainsMembers(int memberCount) { + var page = pageRepository.getOpenPage(bucketId); + assertThat(page.getAvailableMemberSpace()).isEqualTo(memberCount); + } + + @Then("I expect no more unpaged members") + public void iExpectNoMoreUnpagedMembers() { + var unpagedMembersCount = pageMemberEntityRepository.findAll() + .stream() + .filter(entity -> entity.getBucket() == null) + .count(); + assertThat(unpagedMembersCount).isZero(); + } + + @When("I create a page given the mutable page") + public void iCreateAPageGivenTheMutablePage() { + openPage = pageRepository.getOpenPage(bucketId); + pageRepository.createNextPage(openPage); + } + + @And("The old page has a generic relation to the new page") + public void theOldPageHasAGenericRelationToTheNewPage() { + var oldOpenPageId = openPage.getId(); + openPage = pageRepository.getOpenPage(bucketId); + var newOpenPageId = openPage.getId(); + + var relationPresent = pageRelationEntityRepository.findAll() + .stream() + .anyMatch(pageRelationEntity -> + pageRelationEntity.getRelationId().getFromPageId().equals(oldOpenPageId) && + pageRelationEntity.getRelationId().getToPageId().equals(newOpenPageId) && + pageRelationEntity.getTreeRelationType().equals("https://w3id.org/tree#Relation")); + + assertThat(relationPresent).isTrue(); + } + + private void saveMembers(List memberIds) { + var collectionId = eventStreamEntityRepository.findAll().getFirst().getId(); + String eventStream = "http://example.com/es"; + String sql = "INSERT INTO members (subject, collection_id, version_of, timestamp, transaction_id, member_model, old_id) VALUES (?,?,?,?,?,?,?)"; + + + final List batchArgs = memberIds.stream() + .map(member -> new Object[]{ + "http://example.com/es/%d".formatted(member), collectionId, eventStream, + LocalDateTime.now(), 0, new byte[]{}, + "%s/%s/%d".formatted(eventStream, eventStream, member) + }) + .toList(); + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + + private void saveMemberBuckets(List memberIds) { + + String sql = "insert into page_members (bucket_id, member_id, view_id) VALUES (?, ?, ?)"; + + final List batchArgs = memberIds.stream() + .map(member -> new Object[]{ + bucketId, member, viewId + }) + .toList(); + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java new file mode 100644 index 0000000000..47323daaac --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java @@ -0,0 +1,69 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream.repository.EventStreamEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view.repository.ViewEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.BucketEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageMemberEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import io.cucumber.spring.CucumberContextConfiguration; +import io.zonky.test.db.AutoConfigureEmbeddedDatabase; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.jdbc.Sql; + +@CucumberContextConfiguration +@EnableAutoConfiguration +@DataJpaTest +@AutoConfigureEmbeddedDatabase(refresh = AutoConfigureEmbeddedDatabase.RefreshMode.AFTER_EACH_TEST_METHOD) +@ActiveProfiles("postgres-test") +@EntityScan(basePackages = {"be.vlaanderen.informatievlaanderen.ldes.server"}) +@ComponentScan(basePackages = {"be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres", + "be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres", + "be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres", + "be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream" +}) +@ContextConfiguration(classes = {PageEntityRepository.class}) +@EnableJpaRepositories(basePackageClasses = {PageEntityRepository.class, PageMemberEntityRepository.class, PageRelationEntityRepository.class, +MemberEntityRepository.class, BucketEntityRepository.class, ViewEntityRepository.class, EventStreamEntityRepository.class}) +@Sql(value = {"init-paged-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + +@SuppressWarnings("java:S2187") +public class PostgresPaginationIntegrationTest { + + @Autowired + PageRepository pageRepository; + @Autowired + PageMemberRepository pageMemberRepository; + + @Autowired + PageEntityRepository pageEntityRepository; + @Autowired + PageMemberEntityRepository pageMemberEntityRepository; + @Autowired + PageRelationEntityRepository pageRelationEntityRepository; + + @Autowired + JdbcTemplate jdbcTemplate; + + @Autowired + MemberEntityRepository memberRepository; + @Autowired + BucketEntityRepository bucketEntityRepository; + + @Autowired + EventStreamEntityRepository eventStreamEntityRepository; + @Autowired + ViewEntityRepository viewEntityRepository; + +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java deleted file mode 100644 index 2563eb98b1..0000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.database.JdbcBatchItemWriter; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - -@ExtendWith(MockitoExtension.class) -class PageAssignmentsWriterTest { - private static final long BUCKET_ID = 123; - private static final long PAGE_ID = 1234; - private static final long CHILD_PAGE_ID = 12345; - - @Mock - private JdbcBatchItemWriter delegateWriter; - private PageAssignmentsWriter pageAssignmentsWriter; - @Captor - private ArgumentCaptor> captor; - - @BeforeEach - public void setup() { - pageAssignmentsWriter = new PageAssignmentsWriter(delegateWriter); - } - - @Test - void test_write() throws Exception { - Chunk> chunk = Chunk.of( - List.of(new PageAssignment(PAGE_ID, BUCKET_ID, 1), - new PageAssignment(PAGE_ID, BUCKET_ID, 2)), - List.of(new PageAssignment(CHILD_PAGE_ID, BUCKET_ID, 3), - new PageAssignment(CHILD_PAGE_ID, BUCKET_ID, 4)) - ); - - pageAssignmentsWriter.write(chunk); - - verify(delegateWriter).write(captor.capture()); - assertThat(captor.getValue().size()).isEqualTo(4); - } - - @Test - void given_emptyChunk_test_write() throws Exception { - pageAssignmentsWriter.write(new Chunk<>()); - - verifyNoInteractions(delegateWriter); - } - - @Test - void given_chunkWithEmptyLists_test_write() throws Exception { - pageAssignmentsWriter.write(Chunk.of(List.of())); - - verifyNoInteractions(delegateWriter); - } -} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java deleted file mode 100644 index db7dd6b823..0000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch.PageRelationProcessor; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.Collections; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -class PageRelationProcessorTest { - private static final int PAGE_SIZE = 150; - private static final long BUCKET_ID = 12; - private static final int UNPROCESSED_MEMBER_COUNT = 32; - private static final long PARENT_PAGE_ID = 2; - private static final long CHILD_PAGE_ID = PARENT_PAGE_ID + 1; - - @Mock - private PageRepository pageRepository; - @Mock - private PageRelationRepository pageRelationRepository; - @InjectMocks - private PageRelationProcessor pageRelationProcessor; - - @Test - void given_FullPage_when_Process_then_CreateNewPage() { - final Page parentPage = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page?pageNumber=2", PAGE_SIZE, PAGE_SIZE); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(parentPage); - when(pageRepository.createPage(BUCKET_ID, parentPage.createChildPartialUrl().asString())).thenReturn((int) CHILD_PAGE_ID); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - verify(pageRepository).createPage(parentPage.getBucketId(), parentPage.createChildPartialUrl().asString()); - verify(pageRelationRepository).insertGenericBucketRelation(eq(parentPage.getId()), anyLong()); - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, CHILD_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT); - } - - @Test - void given_RootPage_when_Process_then_CreateNewPage() { - final Page rootPage = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page", PAGE_SIZE, 0); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(rootPage); - when(pageRepository.createPage(BUCKET_ID, rootPage.createChildPartialUrl().asString())).thenReturn((int) CHILD_PAGE_ID); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - verify(pageRepository).createPage(rootPage.getBucketId(), rootPage.createChildPartialUrl().asString()); - verify(pageRelationRepository).insertGenericBucketRelation(eq(rootPage.getId()), anyLong()); - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, CHILD_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT); - } - - @Test - void given_PageWithSpace_when_Process_then_UseExistingPage() { - final Page rootPage = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page?pageNumber=3", PAGE_SIZE, 100); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(rootPage); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, PARENT_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT); - } - - @Test - void given_PageWithInsufficientSpace_when_Process_then_UseBothExistingAndNewPage() { - final int alreadyAssignedMemberCount = 132; - final Page page = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page?pageNumber=3", PAGE_SIZE, alreadyAssignedMemberCount); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(page); - when(pageRepository.createPage(BUCKET_ID, page.createChildPartialUrl().asString())).thenReturn((int) CHILD_PAGE_ID); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, PARENT_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(PAGE_SIZE - alreadyAssignedMemberCount); - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, CHILD_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT - (PAGE_SIZE - alreadyAssignedMemberCount)); - } - - @Test - void given_ZeroAssignedMembers_when_Process_then_ReturnEmptyList() { - final List result = pageRelationProcessor.process(List.of()); - - assertThat(result).isEmpty(); - } -} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml new file mode 100644 index 0000000000..b8c8bf7257 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml @@ -0,0 +1,3 @@ +spring: + liquibase: + change-log: classpath:/db/changelog/master.xml \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql new file mode 100644 index 0000000000..47619a32aa --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql @@ -0,0 +1,17 @@ +WITH new_collection AS ( + INSERT INTO collections (name, timestamp_path, version_of_path, create_versions, is_closed, skolemization_domain) + VALUES ('es', '', '', false, false, null) + RETURNING collection_id +), + new_view AS ( + INSERT INTO views (collection_id, name, fragmentations, retention_policies, page_size) + SELECT collection_id, 'es/view', '[]', '[]', 10 FROM new_collection + RETURNING view_id + ), + new_bucket AS ( + INSERT INTO buckets (bucket, view_id) + SELECT '', view_id FROM new_view + RETURNING bucket_id + ) +INSERT INTO pages (bucket_id, partial_url, immutable) +SELECT bucket_id, '/es/view?pageNumber=1', false FROM new_bucket; \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature new file mode 100644 index 0000000000..c99c34f6a5 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature @@ -0,0 +1,19 @@ +Feature: Pagination Interactions + + Scenario: Assign members to page (5/10 members) + Given I have 5 unpaged members in one bucket + When I assign the members to the page + Then I expect no more unpaged members + And the open page has space for 5 more members + + Scenario: Assign members to page (10/10 members) + Given I have 10 unpaged members in one bucket + When I assign the members to the page + And I expect no more unpaged members + + Scenario: Create new page + When I create a page given the mutable page + Then I expect a new page is created + And The old page has a generic relation to the new page + + diff --git a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml index 643745f7c8..403e150fd7 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0 + 3.4.0-SNAPSHOT postgres-retention-repository diff --git a/ldes-server-instrumentation/pom.xml b/ldes-server-instrumentation/pom.xml index f495e26df4..b2b1f472ae 100644 --- a/ldes-server-instrumentation/pom.xml +++ b/ldes-server-instrumentation/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-instrumentation diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index baeb77a279..2ed29db813 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-integration-test diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java index 52ae3456df..46b872d426 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java @@ -1,5 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; @@ -14,6 +16,7 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; @@ -62,7 +65,7 @@ public void verifyCreationOfTheFollowingFragments(int i) { @And("verify the following pages have no relation pointing to them") public void verifyUpdateOfPredecessorRelations(List ids) { await().untilAsserted(() -> { - var count = relationEntityRepository.findAll() + var count = pageRelationEntityRepository.findAll() .stream() .filter(relationEntity -> ids.contains(relationEntity.getToPage().getId())) .count(); @@ -83,14 +86,18 @@ public void verifyRemovalOfPages(List ids) { }); } - @And("verify {long} pages have a relation pointing to the new page {long}") - public void verifyUpdateOfPredecessorRelations(long pointingCount, long id) { + @And("verify {long} pages have a relation pointing to a compacted page") + public void verifyUpdateOfPredecessorRelations(long pointingCount) { await().untilAsserted(() -> { - var countNewPage = relationEntityRepository.findAll() + var countNewPage = pageRelationEntityRepository.findAll() .stream() - .filter(relationEntity -> relationEntity.getToPage().getId().equals(id)) + .map(PageRelationEntity::getToPage) + .map(PageEntity::getPartialUrl) + .map(url -> url.split("pageNumber=")[1]) + .filter(this::isValidUuid) .count(); + assertThat(countNewPage).isEqualTo(pointingCount); }); } @@ -128,4 +135,13 @@ private String readMemberTemplate(String fileName) throws IOException, URISyntax private String getCurrentTimestamp() { return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.[SSS]'Z'")); } + + private boolean isValidUuid(String pageNumber) { + try { + UUID.fromString(pageNumber); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } } diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java index efe97e1c94..1b067aa8e6 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java @@ -82,7 +82,7 @@ public void iFetchTheNextFragmentThroughTheFirst(String relation) { @Then("this fragment only has {int} {string} relation") public void thisFragmentOnlyHasOne(int expectedRelationCount, String relation) { - await().atMost(Duration.of(FRAGMENTATION_POLLING_RATE, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); int relationCount = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) .toList().size(); @@ -106,7 +106,7 @@ public void thisFragmentIsImmutable() { @And("this fragment contains {int} members") public void thisFragmentContainsMembers(int expectedMemberCount) { - await().atMost(Duration.of(FRAGMENTATION_POLLING_RATE, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); return MemberCounter.countMembers(expectedMemberCount).matches(currentFragment); }); @@ -119,7 +119,7 @@ public void thisFragmentIsNotImmutable() { @And("this fragment has no relations") public void thisFragmentHasNoRelations() { - await().atMost(Duration.of(FRAGMENTATION_POLLING_RATE, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); return !currentFragment.listObjectsOfProperty(createProperty(TREE + "relation")).hasNext(); }); diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java index f31f99127d..dc53e37f55 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java @@ -5,7 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.ingest.repositories.MemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.PageRelationPostgresRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.RelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import io.cucumber.spring.CucumberContextConfiguration; import io.zonky.test.db.AutoConfigureEmbeddedDatabase; import jakarta.persistence.EntityManager; @@ -33,14 +33,12 @@ refresh = AutoConfigureEmbeddedDatabase.RefreshMode.AFTER_EACH_TEST_METHOD, replace = AutoConfigureEmbeddedDatabase.Replace.ANY) @ActiveProfiles("postgres-test") -@ContextConfiguration(classes = { MemberEntityRepository.class, PageRelationPostgresRepository.class, RelationEntityRepository.class }) +@ContextConfiguration(classes = { MemberEntityRepository.class, PageRelationPostgresRepository.class, PageRelationEntityRepository.class }) @ComponentScan(value = { "be.vlaanderen.informatievlaanderen.ldes.server" }) @TestPropertySource(properties = { "ldes-server.fragmentation-cron=*/1 * * * * *", "ldes-server.compaction-cron=*/10 * * * * *", "ldes-server.deletion-cron=*/20 * * * * *", "ldes-server.compaction-duration=PT1S" }) @SuppressWarnings("java:S2187") public class LdesServerIntegrationTest { - static final int FRAGMENTATION_POLLING_RATE = 1000; - @Autowired MockMvc mockMvc; @@ -56,7 +54,7 @@ public class LdesServerIntegrationTest { @Autowired MemberMetricsRepository memberMetricsRepository; @Autowired - RelationEntityRepository relationEntityRepository; + PageRelationEntityRepository pageRelationEntityRepository; @Autowired PageEntityRepository pageEntityRepository; diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java index 71b55d027e..bdc8c39692 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java @@ -135,7 +135,7 @@ private Model readModelFromFile(String fileName) throws URISyntaxException { @Then("I can fetch the TreeNode {string} and it contains {int} members") public void iCanFetchTheTreeNodeAndItContainsMembers(String url, int expectedNumberOfMembers) { await() - .atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + .atMost(60, SECONDS) .untilAsserted(() -> { responseModel = fetchFragment(url); assertNotNull(responseModel); @@ -255,7 +255,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l .listObjectsOfProperty(createProperty("https://w3id.org/tree#node")).next().toString(); - await().atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + await().atMost(60, SECONDS) .until(() -> { Model fragmentPage = fetchFragment(fragmentUrl); @@ -266,7 +266,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l @And("the LDES {string} contains {int} members") public void theLDESContainsMembers(String collection, int expectedMemberCount) { - await().atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + await().atMost(60, SECONDS) .until(() -> jdbcTemplate .queryForObject("SELECT COUNT(*) FROM members m JOIN collections c ON m.collection_id = c.collection_id WHERE c.name = ?", Integer.class, collection) @@ -313,7 +313,7 @@ public void iIngestFilesOfStateObjectsFromFolderToTheCollection(int numberOfStat @When("I fetch a fragment from url {string} in a streaming way and is equal to the model of {string}") public void iFetchAStreamingFragment(String url, String compareUrl) { - await().atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + await().atMost(60, SECONDS) .untilAsserted(() -> { FluxExchangeResult response = client.get() .uri(url) diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java index be7be70a0f..c10de92064 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java @@ -2,6 +2,8 @@ import org.apache.jena.rdf.model.Model; import org.mockito.ArgumentMatcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.test.util.AssertionErrors; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.ResultMatcher; @@ -9,6 +11,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.TREE_MEMBER; public class MemberCounter implements ResultMatcher, ArgumentMatcher { + private static final Logger log = LoggerFactory.getLogger(MemberCounter.class); private final int expectedMemberCount; private MemberCounter(int expectedMemberCount) { @@ -28,6 +31,10 @@ public void match(MvcResult result) throws Exception { @Override public boolean matches(Model model) { - return expectedMemberCount == model.listObjectsOfProperty(TREE_MEMBER).toList().size(); + final int memberCount = model.listObjectsOfProperty(TREE_MEMBER).toList().size(); + if(memberCount != expectedMemberCount) { + log.atInfo().log("Expected {} members, but received {}", expectedMemberCount, memberCount); + } + return expectedMemberCount == memberCount; } } diff --git a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml index 65d8727bdc..b19995f7fc 100644 --- a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml +++ b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml @@ -3,6 +3,7 @@ ldes-server: compaction-duration: "*/10 * * * * *" retention-cron: "*/10 * * * * *" deletion-cron: "*/5 * * * * *" + fragmentation-cron: "*/10 * * * * *" springdoc.swaggerui.path: "/swagger" management: diff --git a/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature b/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature index b5cd64db70..c73719ec3b 100644 --- a/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature +++ b/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature @@ -14,7 +14,7 @@ Feature: Execute CompactionService And verify the following pages have no relation pointing to them | 2 | | 3 | - And verify 3 pages have a relation pointing to the new page 5 + And verify 3 pages have a relation pointing to a compacted page And verify the following pages have no members | 2 | | 3 | diff --git a/ldes-server-pagination/pom.xml b/ldes-server-pagination/pom.xml index f99a74ab58..b92029b8d0 100644 --- a/ldes-server-pagination/pom.xml +++ b/ldes-server-pagination/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-pagination diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java deleted file mode 100644 index df0620701a..0000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java +++ /dev/null @@ -1,82 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PartialUrl; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.stereotype.Component; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; - -@Component -public class PageRelationProcessor implements ItemProcessor, List> { - private final PageRepository pageRepository; - private final PageRelationRepository pageRelationRepository; - - public PageRelationProcessor(PageRepository pageRepository, PageRelationRepository pageRelationRepository) { - this.pageRepository = pageRepository; - this.pageRelationRepository = pageRelationRepository; - } - - @Override - public List process(List items) { - if (items.isEmpty()) { - return List.of(); - } - - Page pageToFill = pageRepository.getOpenPage(items.getFirst().bucketId()); - - Queue unpagedMembers = new ArrayDeque<>(items); - List pageAssignments = new ArrayList<>(); - - while (!unpagedMembers.isEmpty()) { - pageToFill = getPageWithSpace(pageToFill); - int membersAdded = fillPage(pageToFill, unpagedMembers.size()); - for (int i = 0; i < membersAdded; i++) { - UnpagedMember member = unpagedMembers.poll(); - assert member != null; - pageAssignments.add(new PageAssignment(pageToFill.getId(), pageToFill.getBucketId(), member.memberId())); - } - } - - return pageAssignments; - } - - private int fillPage(Page pageToFile, int numOfMembersToAdd) { - int membersAdded = Math.min(numOfMembersToAdd, pageToFile.getAvailableMemberSpace()); - pageToFile.incrementAssignedMemberCount(membersAdded); - return membersAdded; - } - - private Page getPageWithSpace(Page page) { - if (page.isNumberLess() || page.isFull()) { - return createNewPage(page); - } - return page; - } - - - private Page createNewPage(Page page) { - final PartialUrl childPagePartialUrl = page.createChildPartialUrl(); - final long childPageId = pageRepository.createPage(page.getBucketId(), childPagePartialUrl.asString()); - closeParentIfNecessary(page); - addRelation(page.getId(), childPageId); - return new Page(childPageId, page.getBucketId(), childPagePartialUrl, page.getPageSize()); - } - - private void closeParentIfNecessary(Page parentPage) { - if (!parentPage.isNumberLess()) { - pageRepository.setPageImmutable(parentPage.getId()); - } - } - - private void addRelation(long parentPageId, long childPageId) { - pageRelationRepository.insertGenericBucketRelation(parentPageId, childPageId); - } -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java index b7691afaaa..d353a64cd6 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java @@ -1,24 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; import org.springframework.batch.core.Step; import org.springframework.batch.core.partition.support.Partitioner; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.core.task.TaskExecutor; import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionException; - -import java.sql.SQLException; -import java.util.List; @Configuration public class PaginationJobDefinitions { @@ -27,22 +18,14 @@ public class PaginationJobDefinitions { @Bean public Step paginationStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, - Partitioner bucketPartitioner, ItemReader> pageItemReader, - ItemProcessor, List> pageRelationsProcessor, - ItemWriter> memberAssigner, + Partitioner bucketPartitioner, + Paginator paginator, PaginationMetricUpdater paginationMetricUpdater, @Qualifier("paginationTaskExecutor") TaskExecutor taskExecutor) { return new StepBuilder(PAGINATION_STEP, jobRepository) .partitioner("memberBucketPartitionStep", bucketPartitioner) .step(new StepBuilder("paginationStep", jobRepository) - ., List>chunk(1, transactionManager) - .reader(pageItemReader) - .processor(pageRelationsProcessor) - .writer(memberAssigner) - .faultTolerant() - .retryLimit(3) - .retry(SQLException.class) - .retry(TransactionException.class) + .tasklet(paginator, transactionManager) .build() ) .allowStartIfComplete(true) @@ -53,8 +36,6 @@ public Step paginationStep(JobRepository jobRepository, PlatformTransactionManag @Bean("paginationTaskExecutor") public TaskExecutor paginationTaskExecutor() { - var taskExecutor = new SimpleAsyncTaskExecutor("spring_batch"); - taskExecutor.setConcurrencyLimit(5); - return taskExecutor; + return new SimpleAsyncTaskExecutor("spring_batch"); } } diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java new file mode 100644 index 0000000000..69a6664a0f --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java @@ -0,0 +1,65 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class Paginator implements Tasklet { + private final PageMemberRepository pageMemberRepository; + private final PageRepository pageRepository; + + public Paginator(PageMemberRepository pageMemberRepository, PageRepository pageRepository) { + this.pageMemberRepository = pageMemberRepository; + this.pageRepository = pageRepository; + } + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + ExecutionContext context = chunkContext.getStepContext().getStepExecution().getExecutionContext(); + long bucketId = context.getLong("bucketId"); + + List members = pageMemberRepository.getUnpaginatedMembersForBucket(bucketId); + + if (members.isEmpty()) { + return RepeatStatus.FINISHED; + } + + Page openPage = pageRepository.getOpenPage(bucketId); + + if (openPage.isNumberLess()) { + openPage = pageRepository.createNextPage(openPage); + } + + int membersInPage; + + for (int i = 0; i < members.size(); i += membersInPage) { + List pageMembers = members.subList(i, Math.min(i + openPage.getAvailableMemberSpace(), members.size())); + membersInPage = pageMembers.size(); + + openPage = fillPageWithMembers(openPage, pageMembers); + } + + return RepeatStatus.FINISHED; + } + + private Page fillPageWithMembers(Page openPage, List pageMembers) { + openPage.incrementAssignedMemberCount(pageMembers.size()); + pageMemberRepository.assignMembersToPage(openPage, pageMembers); + + if (openPage.isFull()) { + return pageRepository.createNextPage(openPage); + } + else { + return openPage; + } + } +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java new file mode 100644 index 0000000000..37ab7695be --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java @@ -0,0 +1,23 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; + +import java.util.List; + +public interface PageMemberRepository { + /** + * Retrieves a list of unpaginated member ids for a bucket + * + * @param bucketId Id of bucket + * @return List of unpaginated member ids for a bucket + */ + List getUnpaginatedMembersForBucket(long bucketId); + + /** + * Assigns Page Member to a Page + * + * @param openPage Open page that will be used to assign members to + * @param pageMembers A list of member ids that will be assigned to a page + */ + void assignMembersToPage(Page openPage, List pageMembers); +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java index 65cd97ac8b..0339e1e3ed 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java @@ -1,12 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; - import java.util.List; public interface PageRelationRepository { void insertGenericBucketRelation(long fromPageId, long toPageId); - void insertBucketRelation(BucketRelation bucketRelation); void updateCompactionBucketRelations(List compactedPageIds, long targetId); } diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java index c18155afa2..032bd5d0e3 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java @@ -10,9 +10,7 @@ public interface PageRepository { Page getOpenPage(long bucketId); - int createPage(Long bucketId, String partialUrl); - void setPageImmutable(long pageId); - void setChildrenImmutableByBucketId(long bucketId); + Page createNextPage(Page parentPage); void markAllPagesImmutableByCollectionName(String collectionName); Stream getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage); void deleteOutdatedFragments(LocalDateTime deleteTime); diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java deleted file mode 100644 index ce1754fe02..0000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java +++ /dev/null @@ -1,16 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -import java.util.Optional; - -public interface PaginationSequenceRepository { - Optional findLastProcessedSequence(ViewName viewName); - - void saveLastProcessedSequence(FragmentSequence sequence); - - void deleteByViewName(ViewName viewName); - - void deleteByCollection(String collectionName); -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java deleted file mode 100644 index bc090b3337..0000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java +++ /dev/null @@ -1,20 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -@Component -public class BucketRelationsEventListener { - private final PageRelationRepository pageRelationRepository; - - public BucketRelationsEventListener(PageRelationRepository pageRelationRepository) { - this.pageRelationRepository = pageRelationRepository; - } - - @EventListener - public void onBucketRelationCreatedEvent(BucketRelationCreatedEvent event) { - pageRelationRepository.insertBucketRelation(event.bucketRelation()); - } -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java deleted file mode 100644 index 8ceaaeb305..0000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java +++ /dev/null @@ -1,20 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.TimeBasedLinearCachingTriggered; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -@Component -public class LinearCachingEventsListener { - private final PageRepository pageRepository; - - public LinearCachingEventsListener(PageRepository pageRepository) { - this.pageRepository = pageRepository; - } - - @EventListener - public void onLinearCachingTriggered(TimeBasedLinearCachingTriggered event) { - pageRepository.setChildrenImmutableByBucketId(event.bucketId()); - } -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java deleted file mode 100644 index c6650fe8e8..0000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; - -public record Bucket(long id, String descriptor) { -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java deleted file mode 100644 index df69d3ad7d..0000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; - -public record PageAssignment(long pageId, long bucketId, long memberId) { -} diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java new file mode 100644 index 0000000000..e07e51f0a7 --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java @@ -0,0 +1,148 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PartialUrl; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.*; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.scope.context.StepContext; +import org.springframework.batch.item.ExecutionContext; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class PaginatorTest { + private static final long BUCKET_ID = 1; + private static final String VIEW_NAME = "event-stream/paged"; + private static final int PAGE_SIZE = 2; + @Mock + private ChunkContext chunkContext; + @Mock + private PageMemberRepository pageMemberRepository; + @Mock + private PageRepository pageRepository; + @InjectMocks + Paginator paginator; + @Captor + ArgumentCaptor pageCaptor; + @Captor + ArgumentCaptor> listCaptor; + + @Test + void when_Paginating_withNoUnexpectedMembers_doNothing() { + mockBucketId(); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of()); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verifyNoMoreInteractions(); + } + + @Test + void when_Paginating_withNumberedOpenPage_expectCorrectPages() { + mockBucketId(); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(1)), PAGE_SIZE); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); + when(pageRepository.getOpenPage(BUCKET_ID)) + .thenReturn(numberedPage); + + when(pageRepository.createNextPage(any())) + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verify(pageMemberRepository, times(3)).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verifyNoMoreInteractions(); + + assertThat(pageCaptor.getAllValues().stream().map(Page::getId).toList()) + .containsExactlyInAnyOrder(1L, 2L, 3L); + assertThat(listCaptor.getAllValues().stream().map(List::size).toList()) + .containsExactlyInAnyOrder(2,2,1); + } + + @Test + void when_Paginating_withUnNumberedOpenPage_expectCorrectPages() { + mockBucketId(); + Page unNumberedPage = new Page(0, BUCKET_ID, new PartialUrl(VIEW_NAME, "", null), PAGE_SIZE); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", null), PAGE_SIZE); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); + when(pageRepository.getOpenPage(BUCKET_ID)) + .thenReturn(unNumberedPage); + + when(pageRepository.createNextPage(any())) + .thenReturn(numberedPage) + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verify(pageMemberRepository, times(3)).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verifyNoMoreInteractions(); + + assertThat(pageCaptor.getAllValues().stream().map(Page::getId).toList()) + .containsExactlyInAnyOrder(1L, 2L, 3L); + assertThat(listCaptor.getAllValues().stream().map(List::size).toList()) + .containsExactlyInAnyOrder(2,2,1); + } + + @Test + void when_Paginating_withSemiFilledNumberedOpenPage_expectCorrectPages() { + mockBucketId(); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(1)), PAGE_SIZE,1); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); + when(pageRepository.getOpenPage(BUCKET_ID)) + .thenReturn(numberedPage); + when(pageRepository.createNextPage(any())) + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verify(pageMemberRepository).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verify(pageRepository).createNextPage(any()); + inOrder.verify(pageMemberRepository).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verify(pageRepository).createNextPage(any()); + inOrder.verify(pageMemberRepository).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verify(pageRepository).createNextPage(any()); + inOrder.verifyNoMoreInteractions(); + + assertThat(pageCaptor.getAllValues().stream().map(Page::getId).toList()) + .containsExactlyInAnyOrder(1L, 2L, 3L); + assertThat(listCaptor.getAllValues().stream().map(List::size).toList()) + .containsExactlyInAnyOrder(1,2,2); + } + + private void mockBucketId() { + StepContext stepContext = mock(StepContext.class); + StepExecution stepExecution = mock(StepExecution.class); + ExecutionContext executionContext = mock(ExecutionContext.class); + + // Set up the mock behavior + when(chunkContext.getStepContext()).thenReturn(stepContext); + when(stepContext.getStepExecution()).thenReturn(stepExecution); + when(stepExecution.getExecutionContext()).thenReturn(executionContext); + when(executionContext.getLong("bucketId")).thenReturn(BUCKET_ID); + } +} diff --git a/ldes-server-port-fetch-rest/pom.xml b/ldes-server-port-fetch-rest/pom.xml index 50b466e885..109eec0d75 100644 --- a/ldes-server-port-fetch-rest/pom.xml +++ b/ldes-server-port-fetch-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-port-fetch-rest diff --git a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java index c2e1f862b9..9a19024893 100644 --- a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java +++ b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java @@ -1,9 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.caching; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; diff --git a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java index baa9e18bfd..3b8ca530f3 100644 --- a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java +++ b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java @@ -1,8 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.treenode; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.services.StreamingTreeNodeFactory; diff --git a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java index fd73435980..eb77b22493 100644 --- a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java +++ b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.treenode.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import org.apache.jena.datatypes.TypeMapper; import org.apache.jena.datatypes.xsd.XSDDatatype; import org.apache.jena.rdf.model.Property; @@ -60,9 +59,4 @@ private boolean hasMeaningfulValue(String objectContent) { return objectContent != null && !objectContent.isEmpty(); } - public static TreeRelationResponse fromRelation(TreeRelation relation, String prefix) { - return new TreeRelationResponse(relation.treePath(), - prefix + relation.treeNode().asEncodedFragmentId(), - relation.treeValue(), relation.treeValueType(), relation.relation()); - } } diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java index 60d23a4743..d918cc0753 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.caching; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java index 953d631d2a..4746c30e58 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java @@ -11,6 +11,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.services.StreamingTreeNodeFactory; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.services.TreeNodeFetcher; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.rest.caching.CachingStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.rest.caching.EtagCachingStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.rest.config.RestConfig; diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java index 1fea6a552a..8796515ea5 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java @@ -8,6 +8,8 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.rest.PrefixConstructor; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.rdf.model.Resource; diff --git a/ldes-server-port-fetch/pom.xml b/ldes-server-port-fetch/pom.xml index 25145535bf..732383753c 100644 --- a/ldes-server-port-fetch/pom.xml +++ b/ldes-server-port-fetch/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-port-fetch diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java index d4d76dd473..e810136b0f 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import org.jetbrains.annotations.Nullable; import java.time.LocalDateTime; diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/LdesFragmentIdentifierParseException.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/exceptions/LdesFragmentIdentifierParseException.java similarity index 80% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/LdesFragmentIdentifierParseException.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/exceptions/LdesFragmentIdentifierParseException.java index 1946482fa2..5aead1a7d6 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/LdesFragmentIdentifierParseException.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/exceptions/LdesFragmentIdentifierParseException.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.exceptions; public class LdesFragmentIdentifierParseException extends RuntimeException { diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java index d6f32016b8..7413027bdb 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import java.util.Optional; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java index ce5bf42d07..fb3714560b 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java index 6da3e61466..19381dd578 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeMemberRepository; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java index eb63bec2b8..0d9cd50ed7 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; public interface TreeNodeFetcher { diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java index 3b17a44abc..203d5a5cb8 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java @@ -1,8 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java new file mode 100644 index 0000000000..fb05b79d0e --- /dev/null +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java @@ -0,0 +1,4 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; + +public record FragmentPair(String fragmentKey, String fragmentValue) { +} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentIdentifier.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifier.java similarity index 73% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentIdentifier.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifier.java index ae515f729d..d3d97928a9 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentIdentifier.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifier.java @@ -1,9 +1,12 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.LdesFragmentIdentifierParseException; -import org.springframework.data.annotation.PersistenceCreator; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.exceptions.LdesFragmentIdentifierParseException; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import static java.net.URLEncoder.encode; @@ -14,7 +17,6 @@ public class LdesFragmentIdentifier { private final ViewName viewName; private final List fragmentPairs; - @PersistenceCreator public LdesFragmentIdentifier(ViewName viewName, List fragmentPairs) { this.viewName = viewName; this.fragmentPairs = fragmentPairs; @@ -25,20 +27,6 @@ public LdesFragmentIdentifier(String viewName, List fragmentPairs) this.fragmentPairs = fragmentPairs; } - public ViewName getViewName() { - return viewName; - } - - public List getFragmentPairs() { - return fragmentPairs; - } - - public Optional getValueOfFragmentPairKey(String key) { - return fragmentPairs.stream().filter(pair -> pair.fragmentKey().equals(key)) - .map(FragmentPair::fragmentValue) - .findFirst(); - } - public static LdesFragmentIdentifier fromFragmentId(String fragmentId) { try { String[] splitString = fragmentId.substring(1).split("\\?"); @@ -83,17 +71,6 @@ private String getFragmentId(boolean encoded) { return stringBuilder.toString(); } - public Optional getParentId() { - - if (!this.fragmentPairs.isEmpty()) { - List parentPairs = new ArrayList<>(fragmentPairs); - parentPairs.remove(parentPairs.size() - 1); - - return Optional.of(new LdesFragmentIdentifier(viewName, parentPairs)); - } - return Optional.empty(); - } - @Override public boolean equals(Object o) { if (this == o) diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentRequest.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequest.java similarity index 73% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentRequest.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequest.java index 62c4d1a845..0eff39b04f 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentRequest.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequest.java @@ -1,14 +1,12 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import java.util.List; import java.util.Objects; public record LdesFragmentRequest(ViewName viewName, List fragmentPairs) { - public static LdesFragmentRequest createViewRequest(ViewName viewName) { - return new LdesFragmentRequest(viewName, List.of()); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/TreeRelation.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelation.java similarity index 90% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/TreeRelation.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelation.java index 0dbe63ec1d..f9166f35b1 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/TreeRelation.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelation.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; import java.util.Objects; diff --git a/ldes-server-port-fetch/src/main/java/module-info.java b/ldes-server-port-fetch/src/main/java/module-info.java index a277efe4ba..5d665cae1a 100644 --- a/ldes-server-port-fetch/src/main/java/module-info.java +++ b/ldes-server-port-fetch/src/main/java/module-info.java @@ -3,8 +3,9 @@ exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository; exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities; + exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; - requires ldes.domain; + requires ldes.domain; requires ldes.ingest.domain; requires ldes.fragmentation.domain; diff --git a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java index 8236129016..bb657b1349 100644 --- a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java @@ -1,14 +1,13 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import org.apache.jena.rdf.model.ModelFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,8 +42,7 @@ void setUp() { void when_NoFragmentExists_ThenMissingResourceExceptionIsThrown() { LdesFragmentIdentifier id = new LdesFragmentIdentifier(VIEW_NAME, List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); - Fragment fragment = new Fragment(id); - Mockito.when(treeNodeRepository.findTreeNodeWithoutMembers(fragment.getFragmentId())) + Mockito.when(treeNodeRepository.findTreeNodeWithoutMembers(id)) .thenReturn(Optional.empty()); assertThatThrownBy(() -> streamingTreeNodeFactory.getFragmentWithoutMemberData(id)) diff --git a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java index 23e90457da..a252a5dac1 100644 --- a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java @@ -1,16 +1,14 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import java.util.List; import java.util.Optional; @@ -39,10 +37,10 @@ void setUp() { void when_getFragment_WhenNoFragmentExists_ThenMissingResourceExceptionIsThrown() { LdesFragmentRequest ldesFragmentRequest = new LdesFragmentRequest(VIEW_NAME, List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); - Fragment fragment = new Fragment(new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), - ldesFragmentRequest.fragmentPairs())); - Mockito.when(treeNodeRepository.findByFragmentIdentifier(fragment.getFragmentId())) - .thenThrow(new MissingResourceException("TreeNode", fragment.getFragmentIdString())); + LdesFragmentIdentifier ldesFragmentIdentifier = new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), + ldesFragmentRequest.fragmentPairs()); + when(treeNodeRepository.findByFragmentIdentifier(ldesFragmentIdentifier)) + .thenThrow(new MissingResourceException("TreeNode", ldesFragmentIdentifier.asDecodedFragmentId())); assertThatThrownBy(() -> treeNodeFetcher.getFragment(ldesFragmentRequest)) .isInstanceOf(MissingResourceException.class) @@ -53,12 +51,12 @@ void when_getFragment_WhenNoFragmentExists_ThenMissingResourceExceptionIsThrown( void when_getFragment_WhenExactFragmentExists_ThenReturnThatFragment() { LdesFragmentRequest ldesFragmentRequest = new LdesFragmentRequest(VIEW_NAME, List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); - Fragment fragment = new Fragment(new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), - ldesFragmentRequest.fragmentPairs())); - TreeNode treeNode = new TreeNode(fragment.getFragmentIdString(), true, false, List.of(), + LdesFragmentIdentifier ldesFragmentIdentifier = new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), + ldesFragmentRequest.fragmentPairs()); + TreeNode treeNode = new TreeNode(ldesFragmentIdentifier.asDecodedFragmentId(), true, false, List.of(), List.of(), "collectionName", null); - when(treeNodeRepository.findByFragmentIdentifier(fragment.getFragmentId())).thenReturn(Optional.of(treeNode)); + when(treeNodeRepository.findByFragmentIdentifier(ldesFragmentIdentifier)).thenReturn(Optional.of(treeNode)); TreeNode returnedTreeNode = treeNodeFetcher.getFragment(ldesFragmentRequest); diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/LdesFragmentIdentifierTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifierTest.java similarity index 81% rename from ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/LdesFragmentIdentifierTest.java rename to ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifierTest.java index 54f4efc3f8..3ce7b5ba9f 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/LdesFragmentIdentifierTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifierTest.java @@ -1,17 +1,15 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.ldesfragment.valueobjects; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.LdesFragmentIdentifierParseException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.exceptions.LdesFragmentIdentifierParseException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Optional; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; class LdesFragmentIdentifierTest { @@ -91,14 +89,4 @@ void when_NonRootFragmentIdentifier_Then_CreateEncodedFragmentIdString_withOnlyT assertEquals(encodedFragmentIdString, fragmentId.asEncodedFragmentId()); } - @Test - void when_KeyPresent_Then_ReturnKey() { - assertEquals(fragmentPairValue1, fragmentId.getValueOfFragmentPairKey(fragmentPairKey1).get()); - } - - @Test - void when_KeyNotPresent_Then_ReturnEmptyOptional() { - assertEquals(Optional.empty(), fragmentId.getValueOfFragmentPairKey("NotPresent")); - } - } \ No newline at end of file diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest/valueobjects/LdesFragmentRequestTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequestTest.java similarity index 75% rename from ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest/valueobjects/LdesFragmentRequestTest.java rename to ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequestTest.java index 3eeab7b98a..1611bdca80 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest/valueobjects/LdesFragmentRequestTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequestTest.java @@ -1,7 +1,5 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.ldesfragmentrequest.valueobjects; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -14,10 +12,8 @@ import java.util.List; import java.util.stream.Stream; -import static org.apache.commons.lang3.ObjectUtils.isEmpty; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; class LdesFragmentRequestTest { @@ -51,16 +47,7 @@ void test_InequalityOfLdesFragmentRequests(Object otherLdesFragmentRequest) { assertNotEquals(ldesFragmentRequest, otherLdesFragmentRequest); } - @Test - void when_ViewRequestIsCreated_RequestHasViewNameAndEmptyList() { - final ViewName viewName = new ViewName("collection_name", "view_name"); - LdesFragmentRequest request = LdesFragmentRequest.createViewRequest(viewName); - assertEquals(viewName, request.viewName()); - assertTrue(isEmpty(request.fragmentPairs())); - } - - static class LdesFragmentRequestArgumentsProvider implements - ArgumentsProvider { + static class LdesFragmentRequestArgumentsProvider implements ArgumentsProvider { @Override public Stream provideArguments(ExtensionContext context) { diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/TreeRelationTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelationTest.java similarity index 86% rename from ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/TreeRelationTest.java rename to ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelationTest.java index 996c4c69fe..7207a70994 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/TreeRelationTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelationTest.java @@ -1,7 +1,5 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.ldesfragment.valueobjects; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -12,8 +10,7 @@ import java.util.stream.Stream; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.*; class TreeRelationTest { @@ -66,4 +63,4 @@ public Stream provideArguments(ExtensionContext context) { } } -} +} \ No newline at end of file diff --git a/ldes-server-port-ingest-rest/pom.xml b/ldes-server-port-ingest-rest/pom.xml index d001f57c1f..3b2d318eb8 100644 --- a/ldes-server-port-ingest-rest/pom.xml +++ b/ldes-server-port-ingest-rest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-port-ingest-rest diff --git a/ldes-server-port-ingest/pom.xml b/ldes-server-port-ingest/pom.xml index ff07454b25..55ef8001b8 100644 --- a/ldes-server-port-ingest/pom.xml +++ b/ldes-server-port-ingest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-port-ingest diff --git a/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java b/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java index 8d47cfece0..36e2a307a5 100644 --- a/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java +++ b/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest.MembersIngestedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.collection.MemberExtractorCollection; @@ -11,11 +10,9 @@ import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import java.util.List; -import java.util.concurrent.CompletableFuture; import static be.vlaanderen.informatievlaanderen.ldes.server.ingest.constants.IngestConstants.DUPLICATE_MEMBERS_DETECTED; import static be.vlaanderen.informatievlaanderen.ldes.server.ingest.constants.IngestConstants.MEMBER_WITH_ID_INGESTED; @@ -24,18 +21,16 @@ public class MemberIngesterImpl implements MemberIngester { private final MemberIngestValidator validator; private final MemberRepository memberRepository; - private final ApplicationEventPublisher eventPublisher; private final MemberExtractorCollection memberExtractorCollection; private final ServerMetrics serverMetrics; private static final Logger log = LoggerFactory.getLogger(MemberIngesterImpl.class); public MemberIngesterImpl(MemberIngestValidator validator, MemberRepository memberRepository, - ApplicationEventPublisher eventPublisher, MemberExtractorCollection memberExtractorCollection, + MemberExtractorCollection memberExtractorCollection, ServerMetrics serverMetrics) { this.validator = validator; this.memberRepository = memberRepository; - this.eventPublisher = eventPublisher; this.memberExtractorCollection = memberExtractorCollection; this.serverMetrics = serverMetrics; } @@ -53,7 +48,6 @@ public boolean ingest(String collectionName, Model ingestedModel) { log.warn(DUPLICATE_MEMBERS_DETECTED); return false; } - publishIngestedEvent(collectionName, members); serverMetrics.incrementIngestCount(collectionName, ingestedMembersCount); members.forEach(member -> logSuccessfulMemberIngestion(member.getSubject())); return true; @@ -66,15 +60,6 @@ private List extractMembersFromModel(String collectionName, Mode return memberExtractor.extractMembers(model); } - private void publishIngestedEvent(String collectionName, List members) { - CompletableFuture.runAsync(() -> { - final List memberProperties = members.stream() - .map(member -> new MembersIngestedEvent.MemberProperties(member.getCollectionName() + "/" + member.getSubject(), member.getVersionOf(), member.getTimestamp())) - .toList(); - eventPublisher.publishEvent(new MembersIngestedEvent(collectionName, memberProperties)); - }); - } - private void logSuccessfulMemberIngestion(String memberId) { final String loggableMemberId = memberId.replaceAll("[\n\r\t]", "_"); log.debug(MEMBER_WITH_ID_INGESTED, loggableMemberId); diff --git a/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java b/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java index 4241d04303..daa83ebe0d 100644 --- a/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java +++ b/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest.MembersIngestedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.FragmentationMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; @@ -25,7 +24,6 @@ import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; import java.time.ZonedDateTime; @@ -34,7 +32,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) @@ -47,8 +44,6 @@ class MemberIngesterImplTest { @Mock private MemberRepository memberRepository; @Mock - private ApplicationEventPublisher eventPublisher; - @Mock private MemberIngestValidator validator; private MeterRegistry meterRegistry; private MemberIngester memberIngestService; @@ -60,7 +55,7 @@ void setUp() { MemberExtractorCollection memberExtractorCollection = new MemberExtractorCollectionImpl(); ServerMetrics serverMetrics = new ServerMetrics(mock(FragmentationMetricsRepository.class), mock(MemberMetricsRepository.class)); - memberIngestService = new MemberIngesterImpl(validator, memberRepository, eventPublisher, memberExtractorCollection, serverMetrics); + memberIngestService = new MemberIngesterImpl(validator, memberRepository, memberExtractorCollection, serverMetrics); final MemberExtractor memberExtractor = new VersionObjectMemberExtractor(COLLECTION_NAME, "http://purl.org/dc/terms/isVersionOf", "http://www.w3.org/ns/prov#generatedAtTime"); memberExtractorCollection.addMemberExtractor(COLLECTION_NAME, memberExtractor); @@ -82,7 +77,6 @@ void whenValidatorThrowsAnException_thenTheIngestIsAborted_andTheExceptionIsThro var counter = meterRegistry.find(ServerMetrics.INGEST).counter(); assertThat(counter).isNull(); verifyNoInteractions(memberRepository); - verifyNoInteractions(eventPublisher); } @Test @@ -101,7 +95,6 @@ void when_TheMemberAlreadyExists_thenEmptyOptionalIsReturned() { var counter = meterRegistry.find(ServerMetrics.INGEST).counter(); assertThat(counter).isNull(); verify(memberRepository, times(1)).insertAll(List.of(member)); - verifyNoInteractions(eventPublisher); } @Test @@ -120,9 +113,8 @@ void when_TheMemberDoesNotAlreadyExists_thenMemberIsStored() { Gauge counter = meterRegistry.find(ServerMetrics.INGEST).gauge(); assertThat(counter).isNotNull(); assertThat(counter.value()).isEqualTo(1); - InOrder inOrder = inOrder(memberRepository, eventPublisher); + InOrder inOrder = inOrder(memberRepository); inOrder.verify(memberRepository, times(1)).insertAll(List.of(member)); - inOrder.verify(eventPublisher).publishEvent(any(MembersIngestedEvent.class)); inOrder.verifyNoMoreInteractions(); } diff --git a/ldes-server-retention/pom.xml b/ldes-server-retention/pom.xml index 62977ac9b9..f2dddcdbb9 100644 --- a/ldes-server-retention/pom.xml +++ b/ldes-server-retention/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT ldes-server-retention diff --git a/pom.xml b/pom.xml index 9cc9db1ad5..e079614b1a 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0 + 3.4.0-SNAPSHOT pom