From 4a4188902c2006ba63e2107e426f51458c4d7057 Mon Sep 17 00:00:00 2001 From: pj-cegeka <119848850+pj-cegeka@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:55:35 +0100 Subject: [PATCH] Feat: move ldio status to input (#510) Co-authored-by: Yalz --- docs/_ldio/ldio-inputs/ldio-amqp-in.md | 6 +- .../_ldio/ldio-inputs/ldio-archive-file-in.md | 7 +- docs/_ldio/ldio-inputs/ldio-http-in-poller.md | 5 + docs/_ldio/ldio-inputs/ldio-http-in.md | 6 +- docs/_ldio/ldio-inputs/ldio-kafka-in.md | 5 + .../ldio-inputs/ldio-ldes-client-connector.md | 6 + docs/_ldio/ldio-inputs/ldio-ldes-client.md | 10 +- docs/_ldio/pipeline-management/index.md | 18 ++- .../_ldio/pipeline-management/pipeline-api.md | 5 +- .../LdesStructureDiscovererTest.java | 3 +- .../treenodesupplier/MemberSupplier.java | 2 +- .../treenodesupplier/MemberSupplierImpl.java | 5 +- .../treenodesupplier/TreeNodeProcessor.java | 8 +- .../VersionMaterialisedMemberSupplier.java | 5 + .../client/performance/PerformanceTest.java | 7 +- .../ldi/rdf/formatter/JsonLdFrameWriter.java | 3 +- .../ldi/rdf/parser/JenaContextProvider.java | 5 +- .../rdf/parser/JenaContextProviderTest.java | 3 +- .../ldes/ldi/valuobjects/LinkedDataModel.java | 7 +- .../src/test/java/RdfAdapterTest.java | 9 +- .../ClientCredentialsConfig.java | 11 +- .../ldi/processors/LdesClientProcessor.java | 8 +- .../processors/LdesClientProcessorTest.java | 5 +- .../repository/SparqlSelectService.java | 5 +- .../ldes/ldio/components/LdioSender.java | 35 +---- .../ldes/ldio/config/WebConfig.java | 3 +- .../ldio/controller/PipelineController.java | 19 +-- .../controller/PipelineStatusController.java | 2 +- .../ldio/converters/FlattenDeserializer.java | 10 +- .../repositories/PipelineFileRepository.java | 11 +- .../ldio/services/PipelineCreatorService.java | 10 +- .../ldes/ldio/services/PipelineService.java | 30 ++-- .../ldio/services/PipelineStatusService.java | 72 +++++++--- .../valueobjects/ComponentDefinitionTO.java | 3 +- .../InputComponentDefinitionTO.java | 3 +- .../ldes/ldio/valueobjects/PipelineTO.java | 6 +- .../ldes/ldio/MockFlowConfiguration.java | 4 +- .../ldes/ldio/PipelineIntegrationTest.java | 17 --- .../controller/PipelineControllerTest.java | 28 ++++ .../converters/FlattenDeserializerTest.java | 7 +- .../ldes/ldio/modules/DummyIn.java | 20 ++- .../PipelineFileRepositoryTest.java | 7 +- .../ldio/services/PipelineServiceTest.java | 41 ++++++ .../services/PipelineStatusServiceTest.java | 33 +++++ .../ldio/valueobjects/PipelineTOTest.java | 6 +- .../configurator/LdioInputConfigurator.java | 4 +- .../ldes/ldio/events/InputCreatedEvent.java | 6 + .../ldio/events/PipelineDeletedEvent.java | 2 +- .../ldes/ldio/events/PipelineStatusEvent.java | 3 +- .../ldes/ldio/types/LdioInput.java | 50 ++++++- .../ldio/valueobjects/PipelineStatus.java | 2 +- .../ldio/valueobjects/StatusChangeSource.java | 5 + .../ldes/ldio/LdioAmqpIn.java | 29 +++- .../ldio/config/LdioAmqpInAutoConfig.java | 8 +- .../ldes/ldio/AmqpInIntegrationTestSteps.java | 39 ++++- .../ldes/ldio/AmqpIntegrationTest.java | 8 ++ .../ldio/config/LdioAmqpInAutoConfigTest.java | 7 +- .../resources/features/amqp-in-it.feature | 27 +++- .../ldes/ldio/LdioArchiveFileIn.java | 35 ++++- .../config/LdioArchiveFileInAutoConfig.java | 12 +- .../ldes/ldio/ArchiveFileInIT.java | 7 + .../ldes/ldio/ArchiveFileInITSteps.java | 6 +- .../ldes/ldio/LdiAzureBlobOut.java | 7 +- .../ldes/ldio/LdioHttpInputPoller.java | 33 ++++- .../config/LdioHttpInputPollerAutoConfig.java | 7 +- .../ldes/ldio/LdioHttpInputPollerTest.java | 48 +++++-- ...LdioLdioHttpInputPollerAutoConfigTest.java | 12 +- .../ldes/ldio/LdioHttpInController.java | 12 +- .../ldes/ldio/LdioHttpInProcess.java | 23 ++- .../ldio/config/LdioHttpInAutoConfig.java | 12 +- .../ldes/ldio/LdioHttpInputTest.java | 30 +++- .../ldio-connectors/ldio-kafka/pom.xml | 5 + .../ldes/ldio/LdioKafkaIn.java | 134 +++++++++++++----- .../ldio/config/LdioKafkaInAutoConfig.java | 60 +------- .../ldio/listener/LdioKafkaInListener.java | 35 +++++ .../config/KafkaCommonIntegrationSteps.java | 2 +- .../config/KafkaInIntegrationTestSteps.java | 35 +++-- .../ldio/config/KafkaIntegrationTest.java | 10 ++ .../config/KafkaOutIntegrationTestSteps.java | 2 +- .../config/LdioKafkaInAutoConfigTest.java | 19 ++- .../resources/features/kafka-in-it.feature | 25 +++- .../ldes/ldio/LdioLdesClientConnectorApi.java | 10 +- .../LdioLdesClientConnectorAutoConfig.java | 7 +- .../ldio/LdioLdesClientConnectorTest.java | 4 +- .../ldes/ldio/LdioLdesClient.java | 40 +++++- .../ldio/config/LdioLdesClientAutoConfig.java | 10 +- .../ldes/ldio/config/LdesClientInIT.java | 7 + .../ldio/config/LdioLdesClientITSteps.java | 9 +- 88 files changed, 1003 insertions(+), 366 deletions(-) create mode 100644 ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineControllerTest.java create mode 100644 ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java create mode 100644 ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java create mode 100644 ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/InputCreatedEvent.java rename ldi-orchestrator/{ldio-application => ldio-common}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java (60%) rename ldi-orchestrator/{ldio-application => ldio-common}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java (61%) create mode 100644 ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/listener/LdioKafkaInListener.java diff --git a/docs/_ldio/ldio-inputs/ldio-amqp-in.md b/docs/_ldio/ldio-inputs/ldio-amqp-in.md index c4c931bf6..63fdc5b4a 100644 --- a/docs/_ldio/ldio-inputs/ldio-amqp-in.md +++ b/docs/_ldio/ldio-inputs/ldio-amqp-in.md @@ -34,4 +34,8 @@ an [AMQP 1.0 queue](https://www.amqp.org/resources/specifications). password: artemis queue: example content-type: application/ld+json -``` \ No newline at end of file +``` + +## Pausing + +When paused, this component will not receive any messages from the queue and will start syncing with the queue when unpaused. \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-archive-file-in.md b/docs/_ldio/ldio-inputs/ldio-archive-file-in.md index 0dbf4d030..ee46f14b1 100644 --- a/docs/_ldio/ldio-inputs/ldio-archive-file-in.md +++ b/docs/_ldio/ldio-inputs/ldio-archive-file-in.md @@ -16,4 +16,9 @@ Please refer to the [core documentation](../../core/ldi-inputs/file-archiving) f | Property | Description | Required | Default | Example | Supported values | |:-----------------|:----------------------------------|:---------|:----------------------------|:-----------------|:--------------------------------| | archive-root-dir | The root directory of the archive | Yes | N/A | /parcels/archive | Linux (+ Mac) and Windows paths | -| source-format | The source format of the files | No | Deduced from file extension | text/turtle | Any Jena supported format | \ No newline at end of file +| source-format | The source format of the files | No | Deduced from file extension | text/turtle | Any Jena supported format | + +## Pausing + +When paused, this component will stop reading from the archive. +When resumed, it will pick up where it left of, ignoring any changes to the file structure. \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md index 1f30ecaba..de6c61c87 100644 --- a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md +++ b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md @@ -39,6 +39,11 @@ config: When using multiple endpoints, the other config (auth config, interval, etc.) applies to all endpoints. +## Pausing + +When paused, this component will stop making any of the scheduled HTTP-calls. +When resumed, it will restart these calls as if the component had been restarted, meaning any configured periods will start counting from the moment the pipeline was resumed instead of when it was originally created. + ---- [^1]: Either choose the 'cron' option or the 'interval'. However, **the interval property will become deprecated**. diff --git a/docs/_ldio/ldio-inputs/ldio-http-in.md b/docs/_ldio/ldio-inputs/ldio-http-in.md index 53e664c2c..1c74059de 100644 --- a/docs/_ldio/ldio-inputs/ldio-http-in.md +++ b/docs/_ldio/ldio-inputs/ldio-http-in.md @@ -14,4 +14,8 @@ Data can be written to ``http://{hostname}:{port}/{pipeline name}`` ## Config -This component has no required config \ No newline at end of file +This component has no required config + +## Pausing + +When paused, this component will return an 503 response to any HTTP-calls it receives \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-kafka-in.md b/docs/_ldio/ldio-inputs/ldio-kafka-in.md index 80cea40f1..9e27a9f80 100644 --- a/docs/_ldio/ldio-inputs/ldio-kafka-in.md +++ b/docs/_ldio/ldio-inputs/ldio-kafka-in.md @@ -53,3 +53,8 @@ outputs: sasl-jaas-user: client sasl-jaas-password: client-secret ``` + +## Pausing + +When paused, this component will stop listening to the kafka topics. +When resumed, it will try to resync with all topics. \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md index adf8ad2e8..ec9d52f5c 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md @@ -53,3 +53,9 @@ input: proxy-url-replacement: http://consumer-connector:29291/public source-format: application/n-quads ``` + +## Pausing + +When paused, this component will stop processing the current fragment and not make any calls to the server. +When resumed, it will continue with the fragment where it stopped and continue as normal. +This component can not be paused while waiting for the token. \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index ef4e93283..c2e704815 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -40,6 +40,9 @@ An LDIO wrapper component for the [LDI LDES Client building block](../../core/ld This component uses the "LDIO Http Requester" to make the HTTP request. Refer to [LDIO Http Requester](../ldio-core) for the config. +> **_NOTE:_** Setting the keep-state property to true makes it so that the state can not be deleted through the pipeline-management api + + ## Examples ```yaml @@ -74,4 +77,9 @@ Refer to [LDIO Http Requester](../ldio-core) for the config. url: jdbc:postgresql://test.postgres.database.azure.com:5432/sample username: myUsername@test password: myPassword -``` \ No newline at end of file +``` + +## Pausing + +When paused, this component will stop processing the current fragment and not make any calls to the server. +When resumed, it will continue with the fragment where it stopped and continue as normal. \ No newline at end of file diff --git a/docs/_ldio/pipeline-management/index.md b/docs/_ldio/pipeline-management/index.md index 93944255f..a04765142 100644 --- a/docs/_ldio/pipeline-management/index.md +++ b/docs/_ldio/pipeline-management/index.md @@ -50,4 +50,20 @@ orchestrator: If this directory does not exist, it will be created. > **_NOTE:_** An application config can be defined by creating an application YAML file in the LDIO directory -(in docker, this correlates to `/ldio/application.yml`). \ No newline at end of file +(in docker, this correlates to `/ldio/application.yml`). + + +## Pausing & Resuming LDIO + +Sometimes it might be preferred to pause an LDIO pipeline instead of deleting and recreating it. +To pause a pipeline, simply call the following endpoint: +```` + {base-url}/admin/api/v1/pipeline/{pipeline-name}/halt +```` +And to resume a paused pipeline: +```` + {base-url}/admin/api/v1/pipeline/{pipeline-name}/resume +```` + +The exact behaviour of a paused pipeline depends on its input component and can be found in the [documentation of these components](docs/_ldio/ldio-inputs/index.md). +However, it will always complete its current run through the pipeline and then seize sending any output. \ No newline at end of file diff --git a/docs/_ldio/pipeline-management/pipeline-api.md b/docs/_ldio/pipeline-management/pipeline-api.md index 0f4c500d7..7c2d08145 100644 --- a/docs/_ldio/pipeline-management/pipeline-api.md +++ b/docs/_ldio/pipeline-management/pipeline-api.md @@ -90,4 +90,7 @@ A deletion of a pipeline can be achieved by performing a DELETE request to the ````bash curl --location --request DELETE 'http://localhost:8080/admin/api/v1/pipeline/my-first-pipeline' -```` \ No newline at end of file +```` + +For stateful components like the Ldio ldes-client it is possible to persist the state on the removal of the pipeline. +Keeping the state is the default option. If you want to specifically delete the state simply send false in the request body of the delete statement. \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java index 5c363cdcb..9ef3dbfff 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; import ldes.client.treenoderelationsfetcher.domain.valueobjects.LdesRelation; import ldes.client.treenoderelationsfetcher.domain.valueobjects.LdesStructure; import org.apache.commons.io.FileUtils; @@ -17,6 +16,8 @@ import java.util.Objects; import java.util.stream.Stream; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + import static com.github.tomakehurst.wiremock.client.WireMock.*; import static org.assertj.core.api.Assertions.assertThat; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplier.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplier.java index afd88c915..fb7e42691 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplier.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplier.java @@ -8,7 +8,7 @@ public interface MemberSupplier extends Supplier { @Override SuppliedMember get(); - + void init(); void destroyState(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java index b76171b48..6a8734ed2 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java @@ -10,7 +10,6 @@ public class MemberSupplierImpl implements MemberSupplier { public MemberSupplierImpl(TreeNodeProcessor treeNodeProcessor, boolean keepState) { this.treeNodeProcessor = treeNodeProcessor; this.keepState = keepState; - Runtime.getRuntime().addShutdownHook(new Thread(this::destroyState)); } @Override @@ -25,4 +24,8 @@ public void destroyState() { } } + @Override + public void init() { + treeNodeProcessor.init(); + } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java index ea2a4f237..1b15bff6d 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java @@ -34,11 +34,15 @@ public TreeNodeProcessor(LdesMetaData ldesMetaData, StatePersistence statePersis this.ldesMetaData = ldesMetaData; } - public SuppliedMember getMember() { - savePreviousState(); + public void init() { if (!treeNodeRecordRepository.containsTreeNodeRecords()) { initializeTreeNodeRecordRepository(); } + } + + public SuppliedMember getMember() { + savePreviousState(); + Optional unprocessedTreeMember = memberRepository.getUnprocessedTreeMember(); while (unprocessedTreeMember.isEmpty()) { processTreeNode(); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/VersionMaterialisedMemberSupplier.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/VersionMaterialisedMemberSupplier.java index c8cdbf5f6..9c8b35797 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/VersionMaterialisedMemberSupplier.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/VersionMaterialisedMemberSupplier.java @@ -30,4 +30,9 @@ public void destroyState() { memberSupplier.destroyState(); } + @Override + public void init() { + memberSupplier.init(); + } + } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java index 685d1089b..4700d2211 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java @@ -1,8 +1,5 @@ package ldes.client.performance; -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.core.WireMockConfiguration; -import com.github.tomakehurst.wiremock.extension.responsetemplating.ResponseTemplateTransformer; import ldes.client.performance.csvwriter.CsvFile; import ldes.client.treenodesupplier.TreeNodeProcessor; import org.junit.jupiter.api.*; @@ -11,6 +8,10 @@ import java.time.temporal.ChronoUnit; import java.util.List; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.extension.responsetemplating.ResponseTemplateTransformer; + import static org.apache.commons.io.FilenameUtils.separatorsToSystem; /** diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java index 80c92b973..a7282f7b4 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter; -import com.github.jsonldjava.core.JsonLdOptions; import org.apache.jena.atlas.json.JSON; import org.apache.jena.atlas.json.JsonObject; import org.apache.jena.rdf.model.Model; @@ -10,6 +9,8 @@ import java.io.OutputStream; +import com.github.jsonldjava.core.JsonLdOptions; + import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.PrefixAdder.addPrefixesToModel; public class JsonLdFrameWriter implements LdiRdfWriter { diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProvider.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProvider.java index e2b1b754d..fb9549418 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProvider.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProvider.java @@ -1,11 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.parser; -import com.apicatalog.jsonld.JsonLdOptions; -import com.apicatalog.jsonld.context.cache.LruCache; import org.apache.jena.riot.RIOT; import org.apache.jena.sparql.util.Context; import org.apache.jena.sparql.util.ContextAccumulator; +import com.apicatalog.jsonld.JsonLdOptions; +import com.apicatalog.jsonld.context.cache.LruCache; + import static org.apache.jena.riot.lang.LangJSONLD11.JSONLD_OPTIONS; public class JenaContextProvider { diff --git a/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java index 91aa0b344..e68d496d9 100644 --- a/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java +++ b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java @@ -1,9 +1,10 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.parser; +import org.junit.jupiter.api.Test; + import com.apicatalog.jsonld.JsonLdOptions; import com.apicatalog.jsonld.context.cache.Cache; import com.apicatalog.jsonld.document.Document; -import org.junit.jupiter.api.Test; import static org.apache.jena.riot.lang.LangJSONLD11.JSONLD_OPTIONS; import static org.assertj.core.api.Assertions.assertThat; diff --git a/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java b/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java index 6c7eb3669..aaf9824c4 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java +++ b/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java @@ -4,9 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.parser.JenaContextProvider; import be.vlaanderen.informatievlaanderen.ldes.ldi.valuobjects.properties.LinkedDataAttribute; import be.vlaanderen.informatievlaanderen.ldes.ldi.valuobjects.valueproperties.DateTimeValue; -import com.fasterxml.jackson.annotation.*; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.riot.Lang; @@ -17,6 +14,10 @@ import java.util.List; import java.util.Objects; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + import static be.vlaanderen.informatievlaanderen.ldes.ldi.config.NgsiV2ToLdMapping.*; import static be.vlaanderen.informatievlaanderen.ldes.ldi.services.NgsiLdURIParser.toNgsiLdUri; diff --git a/ldi-core/rdf-adapter/src/test/java/RdfAdapterTest.java b/ldi-core/rdf-adapter/src/test/java/RdfAdapterTest.java index ee9338525..67361216e 100644 --- a/ldi-core/rdf-adapter/src/test/java/RdfAdapterTest.java +++ b/ldi-core/rdf-adapter/src/test/java/RdfAdapterTest.java @@ -1,9 +1,5 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import com.apicatalog.jsonld.JsonLdOptions; -import com.apicatalog.jsonld.context.cache.LruCache; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParserBuilder; @@ -12,6 +8,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import com.apicatalog.jsonld.JsonLdOptions; +import com.apicatalog.jsonld.context.cache.LruCache; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static org.apache.jena.riot.lang.LangJSONLD11.JSONLD_OPTIONS; diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java index df0d293d2..530b461b2 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java @@ -2,17 +2,18 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutorSupplier; +import org.apache.http.Header; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; + +import java.util.Collection; + import com.github.scribejava.core.builder.ServiceBuilder; import com.github.scribejava.core.builder.api.DefaultApi20; import com.github.scribejava.core.oauth.OAuth20Service; import com.github.scribejava.core.oauth2.clientauthentication.ClientAuthentication; import com.github.scribejava.core.oauth2.clientauthentication.RequestBodyAuthenticationScheme; import com.github.scribejava.httpclient.apache.ApacheHttpClient; -import org.apache.http.Header; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; - -import java.util.Collection; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.commons.lang3.Validate.notNull; diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index a5c9ba20b..3bcb79d82 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -61,6 +61,7 @@ public class LdesClientProcessor extends AbstractProcessor { private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); private boolean hasLdesEnded; + private boolean keepState; @Override public Set getRelationships() { @@ -89,7 +90,7 @@ public void onScheduled(final ProcessContext context) { TimestampExtractor timestampExtractor = timestampPath.isBlank() ? new TimestampFromCurrentTimeExtractor() : new TimestampFromPathExtractor(createProperty(timestampPath)); TreeNodeProcessor treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor); - boolean keepState = stateKept(context); + keepState = stateKept(context); if (useVersionMaterialisation(context)) { final var versionOfProperty = createProperty(getVersionOfProperty(context)); final var versionMaterialiser = new VersionMaterialiser(versionOfProperty, restrictToMembers(context)); @@ -101,6 +102,7 @@ public void onScheduled(final ProcessContext context) { memberSupplier = new MemberSupplierImpl(treeNodeProcessor, keepState); } + memberSupplier.init(); determineLdesProperties(ldesMetaData, requestExecutor, context); LOGGER.info("LDES Client processor {} configured to follow (sub)streams {} (expected LDES source format: {})", @@ -180,7 +182,9 @@ private void processNextMember(ProcessContext context, ProcessSession session) { @OnRemoved public void onRemoved() { - memberSupplier.destroyState(); + if (!keepState) { + memberSupplier.destroyState(); + } } public static String convertModelToString(Model model, Lang dataDestinationFormat) { diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java index ca684f1cb..6d5d32004 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java @@ -1,7 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParser; @@ -20,6 +18,9 @@ import java.util.Map; import java.util.stream.Stream; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorRelationships.DATA_RELATIONSHIP; import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/repository/SparqlSelectService.java b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/repository/SparqlSelectService.java index db1446756..7776d22c0 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/repository/SparqlSelectService.java +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/repository/SparqlSelectService.java @@ -1,7 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.repository; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; import org.apache.jena.datatypes.RDFDatatype; import org.apache.jena.query.*; import org.apache.jena.rdf.model.Literal; @@ -10,6 +8,9 @@ import java.util.Iterator; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + public class SparqlSelectService { public JsonArray executeSelect(Model inputModel, String queryString) { diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java index d5f6a3bde..55b91dcf4 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java @@ -1,60 +1,31 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.components; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; -import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import io.micrometer.core.instrument.Metrics; import org.apache.jena.rdf.model.Model; -import org.springframework.context.ApplicationEventPublisher; -import java.util.ArrayDeque; import java.util.List; -import java.util.Queue; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RUNNING; public class LdioSender extends LdioTransformer { - private final ApplicationEventPublisher applicationEventPublisher; - private PipelineStatus pipelineStatus; private final List ldiOutputs; - private final Queue queue = new ArrayDeque<>(); private final String pipelineName; private static final String LDIO_DATA_OUT = "ldio_data_out"; - public LdioSender(String pipelineName, ApplicationEventPublisher applicationEventPublisher, + public LdioSender(String pipelineName, List ldiOutputs) { - this.applicationEventPublisher = applicationEventPublisher; this.ldiOutputs = ldiOutputs; - this.pipelineStatus = RUNNING; this.pipelineName = pipelineName; Metrics.counter(LDIO_DATA_OUT, PIPELINE_NAME, pipelineName).increment(0); } @SuppressWarnings({"java:S131", "java:S1301"}) - public void updateStatus(PipelineStatus statusEvent) { - switch (statusEvent) { - case RESUMING -> { - while (!queue.isEmpty()) { - Metrics.counter(LDIO_DATA_OUT, PIPELINE_NAME, pipelineName).increment(); - ldiOutputs.parallelStream().forEach(ldiOutput -> ldiOutput.accept(queue.poll())); - } - this.pipelineStatus = RUNNING; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING)); - } - case HALTED -> this.pipelineStatus = HALTED; - } - } @Override public void apply(Model model) { - if (pipelineStatus == RUNNING) { - Metrics.counter(LDIO_DATA_OUT, PIPELINE_NAME, pipelineName).increment(); - ldiOutputs.parallelStream().forEach(ldiOutput -> ldiOutput.accept(model)); - } else { - queue.add(model); - } + Metrics.counter(LDIO_DATA_OUT, PIPELINE_NAME, pipelineName).increment(); + ldiOutputs.parallelStream().forEach(ldiOutput -> ldiOutput.accept(model)); } } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java index bf7b0cb21..a8362f029 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import org.springframework.context.annotation.Configuration; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -10,6 +9,8 @@ import java.util.List; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; + @Configuration public class WebConfig implements WebMvcConfigurer { @Override diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineController.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineController.java index 6a9d98599..ca6c48ab0 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineController.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineController.java @@ -28,10 +28,7 @@ public PipelineController(PipelineService pipelineService, PipelineStatusService */ @GetMapping() public List overview() { - return pipelineService.getPipelines() - .stream() - .map(config -> PipelineTO.build(config, pipelineStatusService.getPipelineStatus(config.name()))) - .toList(); + return pipelineService.getPipelines(); } /** @@ -44,7 +41,7 @@ public List overview() { public PipelineTO addPipeline(@RequestBody PipelineConfigTO config) { var pipelineConfig = PipelineConfigTO.fromPipelineConfig(pipelineService.addPipeline(config.toPipelineConfig())); - return PipelineTO.build(pipelineConfig, pipelineStatusService.getPipelineStatus(pipelineConfig.name())); + return PipelineTO.build(pipelineConfig, pipelineStatusService.getPipelineStatus(pipelineConfig.name()), pipelineStatusService.getPipelineStatusChangeSource(config.name())); } /** @@ -56,9 +53,15 @@ public PipelineTO addPipeline(@RequestBody PipelineConfigTO config) { * ResponseEntity.noContent() if the pipeline was not found or could not be deleted. */ @DeleteMapping("/{pipeline}") - public ResponseEntity deletePipeline(@PathVariable String pipeline) { - if (pipelineService.deletePipeline(pipeline)) { - return ResponseEntity.ok().build(); + public ResponseEntity deletePipeline(@PathVariable String pipeline, @RequestBody String body) { + boolean keepState = extractKeepState(body); + if (pipelineService.requestDeletion(pipeline, keepState)) { + return ResponseEntity.accepted().build(); } else return ResponseEntity.noContent().build(); } + + public boolean extractKeepState(String input) { + //Default value should be true + return !input.equalsIgnoreCase("false"); + } } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineStatusController.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineStatusController.java index fb75f7dd5..ad77cd6e5 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineStatusController.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineStatusController.java @@ -26,7 +26,7 @@ public PipelineStatus getPipelineStatus(@PathVariable("pipelineId") String pipel return pipelineStatusService.getPipelineStatus(pipelineId); } - @PostMapping(path = "{pipelineId}/halt") + @PostMapping(path = {"{pipelineId}/halt", "{pipelineId}/siesta"}) public PipelineStatus haltPipeline(@PathVariable("pipelineId") String pipelineId) { return pipelineStatusService.haltRunningPipeline(pipelineId); } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java index df4c3ac99..5dbb9fd64 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java @@ -1,15 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.converters; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; - import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + /** * A custom deserializer that flattens a JSON object into a Map of Strings. * The keys in the map are the JSON paths of the values in the input JSON. diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepository.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepository.java index bb3b84d54..9e5f7fe3d 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepository.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepository.java @@ -5,11 +5,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineAlreadyExistsException; import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineParsingException; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineConfigTO; -import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -24,6 +19,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineConfigTO.fromPipelineConfig; import static java.util.Optional.ofNullable; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java index c025dfcc5..8fc0ac042 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java @@ -10,6 +10,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.SenderCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.InvalidComponentException; import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.InvalidPipelineNameException; @@ -68,18 +69,19 @@ public void initialisePipeline(PipelineConfig config) throws InvalidComponentExc Map inputConfig = new HashMap<>(config.getInput().getConfig().getConfig()); inputConfig.put(ORCHESTRATOR_NAME, orchestratorName); - Object ldiInput = configurator.configure(adapter, executor, new ComponentProperties(pipeLineName, inputName, inputConfig)); + LdioInput ldiInput = configurator.configure(adapter, executor, eventPublisher, new ComponentProperties(pipeLineName, inputName, inputConfig)); registerBean(pipeLineName, ldiInput); + eventPublisher.publishEvent(new InputCreatedEvent(config.getName(), ldiInput)); } catch (NoSuchBeanDefinitionException e) { throw new InvalidComponentException(config.getName(), e.getBeanName()); } } - public void removePipeline(String pipeline) { + public void removePipeline(String pipeline, boolean keepState) { DefaultListableBeanFactory beanRegistry = (DefaultListableBeanFactory) configContext.getBeanFactory(); LdioInput ldioInput = (LdioInput) beanRegistry.getBean(pipeline); - ldioInput.shutdown(); + ldioInput.shutdown(keepState); beanFactory.destroyBean(pipeline); } @@ -94,7 +96,7 @@ private ComponentExecutor componentExecutor(final PipelineConfig pipelineConfig) .map(this::getLdioOutput) .toList(); - LdioSender ldioSender = new LdioSender(pipelineConfig.getName(), eventPublisher, ldiOutputs); + LdioSender ldioSender = new LdioSender(pipelineConfig.getName(), ldiOutputs); List processorChain = new ArrayList<>(ldioTransformers.subList(0, ldioTransformers.size())); diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java index 3aeeda0d6..059a478c5 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java @@ -6,8 +6,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineException; import be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineFileRepository; import be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineRepository; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineConfigTO; -import org.springframework.context.ApplicationEventPublisher; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineTO; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import java.io.File; @@ -15,14 +15,14 @@ @Service public class PipelineService { - private final ApplicationEventPublisher applicationEventPublisher; private final PipelineCreatorService pipelineCreatorService; + private final PipelineStatusService pipelineStatusService; private final PipelineRepository pipelineRepository; - public PipelineService(ApplicationEventPublisher applicationEventPublisher, PipelineCreatorService pipelineCreatorService, + public PipelineService(PipelineCreatorService pipelineCreatorService, PipelineStatusService pipelineStatusService, PipelineFileRepository pipelineRepository) { - this.applicationEventPublisher = applicationEventPublisher; this.pipelineCreatorService = pipelineCreatorService; + this.pipelineStatusService = pipelineStatusService; this.pipelineRepository = pipelineRepository; } @@ -46,17 +46,25 @@ public PipelineConfig addPipeline(PipelineConfig pipeline, File persistedFile) t } } - public List getPipelines() { - return pipelineRepository.getActivePipelines(); + public List getPipelines() { + return pipelineRepository.getActivePipelines() + .stream() + .map(config -> PipelineTO.build(config, pipelineStatusService.getPipelineStatus(config.name()), pipelineStatusService.getPipelineStatusChangeSource(config.name()))) + .toList(); } - public boolean deletePipeline(String pipeline) { + public boolean requestDeletion(String pipeline, boolean keepState) { if (pipelineRepository.exists(pipeline)) { - applicationEventPublisher.publishEvent(new PipelineDeletedEvent(pipeline)); - pipelineCreatorService.removePipeline(pipeline); - return pipelineRepository.delete(pipeline); + pipelineStatusService.stopPipeline(pipeline, keepState); + return true; } else { return false; } } + + @EventListener + public void handleStoppedPipeline(PipelineDeletedEvent deletedEvent) { + pipelineRepository.delete(deletedEvent.pipelineId()); + pipelineCreatorService.removePipeline(deletedEvent.pipelineId(), deletedEvent.keepState()); + } } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java index 1ad90232c..1238fd873 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java @@ -1,9 +1,11 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.services; -import be.vlaanderen.informatievlaanderen.ldes.ldio.components.LdioSender; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; -import be.vlaanderen.informatievlaanderen.ldes.ldio.events.SenderCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; @@ -16,15 +18,16 @@ import java.util.stream.Collectors; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; @Component public class PipelineStatusService { + private final ApplicationEventPublisher eventPublisher; private final Logger logger = LoggerFactory.getLogger(PipelineStatusService.class); - private final ApplicationEventPublisher applicationEventPublisher; private final Map savedPipelines; - public PipelineStatusService(ApplicationEventPublisher applicationEventPublisher) { - this.applicationEventPublisher = applicationEventPublisher; + public PipelineStatusService(ApplicationEventPublisher eventPublisher) { + this.eventPublisher = eventPublisher; this.savedPipelines = new HashMap<>(); } @@ -34,6 +37,12 @@ public PipelineStatus getPipelineStatus(String pipelineName) { .orElseThrow(() -> new IllegalArgumentException("No pipeline defined for name " + pipelineName)); } + public StatusChangeSource getPipelineStatusChangeSource(String pipelineName) { + return Optional.ofNullable(savedPipelines.get(pipelineName)) + .map(SavedPipeline::getLastStatusChangeSource) + .orElseThrow(() -> new IllegalArgumentException("No pipeline defined for name " + pipelineName)); + } + @EventListener public void handlePipelineStatusResponse(PipelineStatusEvent statusEvent) { SavedPipeline currentSavedPipeline = savedPipelines.get(statusEvent.pipelineId()); @@ -41,25 +50,33 @@ public void handlePipelineStatusResponse(PipelineStatusEvent statusEvent) { logger.warn("Non initialized pipeline received status update: {}", statusEvent.pipelineId()); return; } - currentSavedPipeline.updateStatus(statusEvent.status()); + currentSavedPipeline.updateStatus(statusEvent.status(), statusEvent.statusChangeSource()); + } + + @EventListener + public void handlePipelineDeleted(PipelineDeletedEvent deletedEvent) { + this.savedPipelines.remove(deletedEvent.pipelineId()); } @EventListener - public void handleSenderCreated(SenderCreatedEvent event) { - savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioSender(), RUNNING)); + public void handlePipelineCreated(InputCreatedEvent event) { + savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), RUNNING)); } public PipelineStatus resumeHaltedPipeline(String pipelineId) { PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); return switch (pipelineStatus) { - case RUNNING -> RUNNING; case HALTED -> { - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineId, RESUMING)); - savedPipelines.get(pipelineId).getLdioSender().updateStatus(RESUMING); + savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUMING); yield RESUMING; } + case INIT -> INIT; + case STARTING -> STARTING; + case RUNNING -> RUNNING; case RESUMING -> RESUMING; + case STOPPED -> STOPPED; + case STOPPING -> STOPPING; }; @@ -70,16 +87,26 @@ public PipelineStatus haltRunningPipeline(String pipelineId) { return switch (pipelineStatus) { case RUNNING, RESUMING -> { - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineId, HALTED)); - savedPipelines.get(pipelineId).getLdioSender().updateStatus(HALTED); + savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALTED); yield HALTED; } + case INIT -> INIT; + case STARTING -> STARTING; case HALTED -> HALTED; + case STOPPED -> STOPPED; + case STOPPING -> STOPPING; }; } + public PipelineStatus stopPipeline(String pipelineId, boolean keepState) { + LdioInput input = savedPipelines.get(pipelineId).getLdioInput(); + input.updateStatus(STOPPING); + eventPublisher.publishEvent(new PipelineDeletedEvent(pipelineId, keepState)); + return STOPPED; + } + public Map getPipelineStatusOverview() { return savedPipelines.entrySet() .stream() @@ -87,24 +114,31 @@ public Map getPipelineStatusOverview() { } static class SavedPipeline { - private final LdioSender ldioSender; + private final LdioInput ldioInput; private PipelineStatus status; + private StatusChangeSource lastStatusChangeSource; - public SavedPipeline(LdioSender ldioSender, PipelineStatus status) { - this.ldioSender = ldioSender; + public SavedPipeline(LdioInput ldioInput, PipelineStatus status) { + this.ldioInput = ldioInput; this.status = status; + lastStatusChangeSource = AUTO; } - public void updateStatus(PipelineStatus status) { + public void updateStatus(PipelineStatus status, StatusChangeSource statusChangeSource) { this.status = status; + this.lastStatusChangeSource = statusChangeSource; } public PipelineStatus getStatus() { return status; } - public LdioSender getLdioSender() { - return ldioSender; + public LdioInput getLdioInput() { + return ldioInput; + } + + public StatusChangeSource getLastStatusChangeSource() { + return lastStatusChangeSource; } } } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/ComponentDefinitionTO.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/ComponentDefinitionTO.java index 121db16dd..8f30e8fb8 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/ComponentDefinitionTO.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/ComponentDefinitionTO.java @@ -1,10 +1,11 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; import be.vlaanderen.informatievlaanderen.ldes.ldio.converters.FlattenDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.util.Map; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + public record ComponentDefinitionTO(String name, @JsonDeserialize(using = FlattenDeserializer.class) Map config) { public ComponentDefinitionTO(String name, Map config) { diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/InputComponentDefinitionTO.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/InputComponentDefinitionTO.java index f33cef7ad..d63e8eefe 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/InputComponentDefinitionTO.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/InputComponentDefinitionTO.java @@ -1,10 +1,11 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; import be.vlaanderen.informatievlaanderen.ldes.ldio.converters.FlattenDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.util.Map; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + public record InputComponentDefinitionTO(String name, ComponentDefinitionTO adapter, @JsonDeserialize(using = FlattenDeserializer.class) Map config) { public InputComponentDefinitionTO(String name, ComponentDefinitionTO adapter, Map config) { diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java index b9a7720f3..dc20694c9 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java @@ -4,9 +4,9 @@ import java.util.List; import java.util.Optional; -public record PipelineTO(String name, PipelineStatus status, String description, InputComponentDefinitionTO input, +public record PipelineTO(String name, PipelineStatus status, StatusChangeSource updateSource, String description, InputComponentDefinitionTO input, List transformers, List outputs) { - public static PipelineTO build(PipelineConfigTO config, PipelineStatus status) { + public static PipelineTO build(PipelineConfigTO config, PipelineStatus status, StatusChangeSource statusChangeSource) { var input = new InputComponentDefinitionTO(config.input().name(), new ComponentDefinitionTO(config.input().adapter().name(), config.input().adapter().config()), config.input().config()); @@ -18,6 +18,6 @@ public static PipelineTO build(PipelineConfigTO config, PipelineStatus status) { var outputs = config.outputs().stream() .map(componentDefinition -> new ComponentDefinitionTO(componentDefinition.name(), componentDefinition.config())) .toList(); - return new PipelineTO(config.name(), status, config.description(), input, transformers, outputs); + return new PipelineTO(config.name(), status, statusChangeSource, config.description(), input, transformers, outputs); } } diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/MockFlowConfiguration.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/MockFlowConfiguration.java index eec31d9d1..d7f385d90 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/MockFlowConfiguration.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/MockFlowConfiguration.java @@ -12,6 +12,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -46,8 +47,9 @@ static class DummyInConfigurator implements LdioInputConfigurator { @Override public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, + ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { - return new DummyIn(executor, adapter); + return new DummyIn(executor, adapter, applicationEventPublisher); } } diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/PipelineIntegrationTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/PipelineIntegrationTest.java index b9545ea4a..c9a0e76b2 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/PipelineIntegrationTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/PipelineIntegrationTest.java @@ -5,7 +5,6 @@ import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParserBuilder; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -16,8 +15,6 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.web.servlet.MockMvc; -import java.util.stream.IntStream; - import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -59,10 +56,6 @@ void verifyBasicPipelineFlow() { @Test void verifyHaltedPipelineFlow() throws Exception { - // Initial Run - IntStream.range(0, 10) - .forEach(value -> dummyIn.sendData()); - await().until(() -> mockVault.getReceivedObjects().size() == 10); // Halt Pipeline mockMvc.perform(post("http://localhost:8080/admin/api/v1/pipeline/%s/halt".formatted(pipeline))).andExpect(status().isOk()); @@ -71,22 +64,12 @@ void verifyHaltedPipelineFlow() throws Exception { return result.getResponse().getContentAsString().equals("HALTED"); }); - IntStream.range(0, 10) - .forEach(value -> dummyIn.sendData()); - Assertions.assertEquals(10, mockVault.getReceivedObjects().size()); - // Resume Pipeline mockMvc.perform(post("http://localhost:8080/admin/api/v1/pipeline/%s/resume".formatted(pipeline))).andExpect(status().isOk()); await().until(() -> { var result = mockMvc.perform(get("http://localhost:8080/admin/api/v1/pipeline/%s/status".formatted(pipeline))).andReturn(); return result.getResponse().getContentAsString().equals("RUNNING"); }); - - // Whilst resuming add more members to pipeline - IntStream.range(0, 10) - .forEach(value -> dummyIn.sendData()); - - await().until(() -> mockVault.getReceivedObjects().size() == 30); } } diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineControllerTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineControllerTest.java new file mode 100644 index 000000000..8c8bbce4d --- /dev/null +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/controller/PipelineControllerTest.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.controller; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.services.PipelineService; +import be.vlaanderen.informatievlaanderen.ldes.ldio.services.PipelineStatusService; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +class PipelineControllerTest { + + private final PipelineService pipelineService = mock(PipelineService.class); + private final PipelineStatusService pipelineStatusService = mock(PipelineStatusService.class); + private final PipelineController controller = new PipelineController(pipelineService, pipelineStatusService); + @ParameterizedTest + @ValueSource(strings = { "", "nonsense&é", "true", "True" }) + void testKeepStateTrue(String input) { + assertTrue(controller.extractKeepState(input)); + } + @ParameterizedTest + @ValueSource(strings = { "false", "False" }) + void testKeepStateFalse(String input) { + assertFalse(controller.extractKeepState(input)); + } + +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializerTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializerTest.java index 291314ee1..511c38f9e 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializerTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializerTest.java @@ -1,8 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.converters; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -12,6 +9,10 @@ import java.nio.charset.StandardCharsets; import java.util.Map; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java index 3a21fc116..87e494d55 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java @@ -3,12 +3,16 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import org.springframework.context.ApplicationEventPublisher; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; public class DummyIn extends LdioInput { private int counter = 0; - public DummyIn(ComponentExecutor executor, LdiAdapter adapter) { - super("DummyIn", "test", executor, adapter, null); + public DummyIn(ComponentExecutor executor, LdiAdapter adapter, ApplicationEventPublisher applicationEventPublisher) { + super("DummyIn", "test", executor, adapter, null, applicationEventPublisher); + this.updateStatus(STARTING); } public void sendData() { @@ -18,7 +22,17 @@ public void sendData() { } @Override - public void shutdown() { + public void shutdown(boolean keepState) { + + } + + @Override + protected void resume() { + + } + + @Override + protected void pause() { } } diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepositoryTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepositoryTest.java index d4be48f9a..75f3e755c 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepositoryTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/repositories/PipelineFileRepositoryTest.java @@ -7,9 +7,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentDefinition; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.InputComponentDefinition; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineConfigTO; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -23,6 +20,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + import static be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineFileRepository.EXTENSION_YAML; import static be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineFileRepository.EXTENSION_YML; import static org.junit.jupiter.api.Assertions.*; diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java new file mode 100644 index 000000000..bf8d87c0e --- /dev/null +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineFileRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +class PipelineServiceTest { + private final String pipelineName = "pipeline"; + private final PipelineCreatorService pipelineCreatorService = mock(PipelineCreatorService.class); + private final PipelineStatusService pipelineStatusService = mock(PipelineStatusService.class); + private final PipelineFileRepository pipelineRepository = mock(PipelineFileRepository.class); + private PipelineService pipelineService; + + @BeforeEach + void setup() { + pipelineService = new PipelineService(pipelineCreatorService, pipelineStatusService, pipelineRepository); + } + + @Test + void when_StoppingPipeline_Then_MethodsAreCalled() { + when(pipelineRepository.exists(pipelineName)).thenReturn(true); + + boolean result = pipelineService.requestDeletion(pipelineName, false); + + assertEquals(true, result); + verify(pipelineStatusService).stopPipeline(pipelineName, false); + } + @Test + void when_StoppingNonExistingPipeline_Then_NoMethodsAreCalled() { + when(pipelineRepository.exists(pipelineName)).thenReturn(false); + + boolean result = pipelineService.requestDeletion(pipelineName, false); + + assertEquals(false, result); + verifyNoInteractions(pipelineStatusService); + } + +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java new file mode 100644 index 000000000..8bafa2c92 --- /dev/null +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java @@ -0,0 +1,33 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +class PipelineStatusServiceTest { + private final String pipelineName = "pipeline"; + private final LdioInput input = mock(LdioInput.class); + private final ApplicationEventPublisher eventPublisher = mock(ApplicationEventPublisher.class); + private PipelineStatusService pipelineStatusService; + + @BeforeEach + void setup() { + pipelineStatusService = new PipelineStatusService(eventPublisher); + pipelineStatusService.handlePipelineCreated(new InputCreatedEvent(pipelineName, input)); + } + + @Test + void when_StoppingPipeline_Then_MethodsAreCalled() { + PipelineStatus result = pipelineStatusService.stopPipeline(pipelineName, false); + + assertEquals(PipelineStatus.STOPPED, result); + verify(input).updateStatus(PipelineStatus.STOPPING); + } + +} diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTOTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTOTest.java index 985f7adab..5465be5bb 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTOTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTOTest.java @@ -6,6 +6,7 @@ import java.util.Map; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RUNNING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,10 +27,11 @@ private static PipelineConfigTO getPipelineConfig() { void fromPipelineConfig() { PipelineConfigTO createdPipeline = getPipelineConfig(); - PipelineTO pipelineTO = PipelineTO.build(createdPipeline, RUNNING); + PipelineTO pipelineTO = PipelineTO.build(createdPipeline, RUNNING, AUTO); assertEquals("pipeline", pipelineTO.name()); assertEquals(RUNNING, pipelineTO.status()); + assertEquals(AUTO, pipelineTO.updateSource()); assertEquals("in", pipelineTO.input().name()); assertTrue(pipelineTO.input().config().containsKey("type")); assertEquals("adapter", pipelineTO.input().adapter().name()); @@ -42,4 +44,4 @@ void fromPipelineConfig() { assertEquals("output", pipelineTO.outputs().get(0).name()); assertTrue(pipelineTO.outputs().stream().allMatch(cd -> cd.config().containsKey("type"))); } -} +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java index a847819e9..6f3e43c2d 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java @@ -2,8 +2,10 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import org.springframework.context.ApplicationEventPublisher; public interface LdioInputConfigurator { - Object configure(LdiAdapter adapter, ComponentExecutor executor, ComponentProperties properties); + LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher eventPublisher, ComponentProperties properties); } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/InputCreatedEvent.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/InputCreatedEvent.java new file mode 100644 index 000000000..ba7d9d77f --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/InputCreatedEvent.java @@ -0,0 +1,6 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.events; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; + +public record InputCreatedEvent(String pipelineName, LdioInput ldioInput) { +} diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineDeletedEvent.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineDeletedEvent.java index 17fc938e6..4af3d4c87 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineDeletedEvent.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineDeletedEvent.java @@ -1,4 +1,4 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.events; -public record PipelineDeletedEvent(String pipelineId) { +public record PipelineDeletedEvent(String pipelineId, boolean keepState) { } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java similarity index 60% rename from ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java rename to ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java index 7c6985e0b..0518ee700 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineStatusEvent.java @@ -1,6 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.events; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource; -public record PipelineStatusEvent(String pipelineId, PipelineStatus status) { +public record PipelineStatusEvent(String pipelineId, PipelineStatus status, StatusChangeSource statusChangeSource) { } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java index 3c2bbffba..9260e43d1 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java @@ -4,14 +4,20 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.ObserveConfiguration; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import io.micrometer.core.instrument.Metrics; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.MANUAL; /** * Base class for the start of a LDIO workflow. @@ -26,11 +32,13 @@ public abstract class LdioInput implements LdiComponent { protected final String pipelineName; private final ComponentExecutor executor; private final LdiAdapter adapter; + private final ApplicationEventPublisher applicationEventPublisher; private final Logger log = LoggerFactory.getLogger(this.getClass()); private final ObservationRegistry observationRegistry; private static final String LDIO_DATA_IN = "ldio_data_in"; private static final String LDIO_COMPONENT_NAME = "ldio_type"; + private PipelineStatus pipelineStatus; /** * Creates a LdiInput with its Component Executor and LDI Adapter @@ -40,12 +48,15 @@ public abstract class LdioInput implements LdiComponent { * @param adapter Instance of the LDI Adapter. Facilitates transforming the input * data to a linked data model (RDF). */ - protected LdioInput(String componentName, String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry) { + protected LdioInput(String componentName, String pipelineName, ComponentExecutor executor, LdiAdapter adapter, + ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher) { this.componentName = componentName; this.pipelineName = pipelineName; this.executor = executor; this.adapter = adapter; this.observationRegistry = observationRegistry; + this.applicationEventPublisher = applicationEventPublisher; + this.pipelineStatus = INIT; Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(0); } @@ -53,7 +64,7 @@ public void processInput(String content, String contentType) { processInput(LdiAdapter.Content.of(content, contentType)); } - protected void processInput(LdiAdapter.Content content) { + public void processInput(LdiAdapter.Content content) { Observation.createNotStarted(this.componentName, observationRegistry) .observe(() -> { try { @@ -81,5 +92,38 @@ protected void processModel(Model model) { }); } - public abstract void shutdown(); + public abstract void shutdown(boolean keepState); + + public void updateStatus(PipelineStatus statusEvent) { + switch (statusEvent) { + case STARTING -> { + this.pipelineStatus = RUNNING; + applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING, AUTO)); + } + case RESUMING -> { + this.resume(); + this.pipelineStatus = RUNNING; + applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING, MANUAL)); + } + case HALTED -> { + if (this.pipelineStatus != INIT) { + this.pause(); + this.pipelineStatus = HALTED; + applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, HALTED, MANUAL)); + } + } + case STOPPING -> { + this.pipelineStatus = STOPPED; + applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, STOPPED, MANUAL)); + } + default -> log.warn("Unhandled status update on pipeline: {} for status: {}", pipelineName, statusEvent); + } + } + + protected abstract void resume(); + protected abstract void pause(); + + public boolean isHalted() { + return pipelineStatus.equals(HALTED) || pipelineStatus.equals(STOPPED); + } } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java similarity index 61% rename from ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java rename to ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java index 6596457da..20228b5fa 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java @@ -1,5 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; public enum PipelineStatus { - RESUMING, RUNNING, HALTED + INIT, STARTING, RESUMING, RUNNING, HALTED, STOPPING, STOPPED } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java new file mode 100644 index 000000000..3b8c86c49 --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java @@ -0,0 +1,5 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; + +public enum StatusChangeSource { + AUTO, MANUAL +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java index c0ccae780..99cfa8784 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java @@ -12,13 +12,17 @@ import jakarta.jms.MessageListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.jms.config.SimpleJmsListenerEndpoint; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.AmqpConfig.CONTENT_TYPE_HEADER; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; public class LdioAmqpIn extends LdioInput implements MessageListener { public static final String NAME = "Ldio:AmqpIn"; private static final Logger log = LoggerFactory.getLogger(LdioAmqpIn.class); + private final LdioAmqpInRegistrator ldioAmqpInRegistrator; + private final String listenerId; private final String defaultContentType; /** @@ -34,10 +38,15 @@ public class LdioAmqpIn extends LdioInput implements MessageListener { * @param observationRegistry */ public LdioAmqpIn(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, - String defaultContentType, JmsConfig jmsConfig, LdioAmqpInRegistrator jmsInRegistrator, ObservationRegistry observationRegistry) { - super(NAME, pipelineName, executor, adapter, observationRegistry); + String defaultContentType, JmsConfig jmsConfig, LdioAmqpInRegistrator jmsInRegistrator, + ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher) { + super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); this.defaultContentType = defaultContentType; - jmsInRegistrator.registerListener(jmsConfig, listenerEndpoint(jmsConfig.queue())); + SimpleJmsListenerEndpoint endpoint = listenerEndpoint(jmsConfig.queue()); + ldioAmqpInRegistrator = jmsInRegistrator; + listenerId = endpoint.getId(); + jmsInRegistrator.registerListener(jmsConfig, endpoint); + this.updateStatus(STARTING); } @Override @@ -63,7 +72,17 @@ private SimpleJmsListenerEndpoint listenerEndpoint(String queue) { } @Override - public void shutdown() { - // Implemented in other story + public void shutdown(boolean keepState) { + ldioAmqpInRegistrator.stopListener(listenerId); + } + + @Override + protected void resume() { + ldioAmqpInRegistrator.startListener(listenerId); + } + + @Override + protected void pause() { + ldioAmqpInRegistrator.stopListener(listenerId); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java index 552b6e892..534c1c42d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java @@ -4,10 +4,12 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -36,7 +38,7 @@ public LdioJmsInConfigurator(LdioAmqpInRegistrator ldioAmqpInRegistrator, Observ } @Override - public Object configure(LdiAdapter adapter, ComponentExecutor executor, ComponentProperties config) { + public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { String pipelineName = config.getPipelineName(); verifyAdapterPresent(pipelineName, adapter); @@ -44,8 +46,8 @@ public Object configure(LdiAdapter adapter, ComponentExecutor executor, Componen JmsConfig jmsConfig = new JmsConfig(config.getProperty(USERNAME), config.getProperty(PASSWORD), remoteUrl, config.getProperty(QUEUE)); - return new LdioAmqpIn(pipelineName, executor, adapter, getContentType(config), jmsConfig, ldioAmqpInRegistrator, observationRegistry - ); + return new LdioAmqpIn(pipelineName, executor, adapter, getContentType(config), jmsConfig, ldioAmqpInRegistrator, + observationRegistry, applicationEventPublisher); } private String getContentType(ComponentProperties config) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java index febbafed0..0da71b9e3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java @@ -6,17 +6,22 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioAmqpInAutoConfig; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioAmqpInRegistrator; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.cucumber.java.ParameterType; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; import jakarta.jms.JMSException; import jakarta.jms.MessageProducer; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.RDFParser; import org.apache.jena.riot.RDFParserBuilder; import org.apache.jena.riot.RDFWriter; +import org.awaitility.Awaitility; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -24,6 +29,8 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn.NAME; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RESUMING; import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; import static org.apache.jena.riot.RDFLanguages.nameToLang; import static org.awaitility.Awaitility.await; @@ -33,6 +40,7 @@ public class AmqpInIntegrationTestSteps extends AmqpIntegrationTest { private final LdioAmqpInRegistrator ldioAmqpInRegistrator = jmsInRegistrator(); + private LdioInput ldioInput; private Model inputModel; private String contentType; private Map config; @@ -62,7 +70,7 @@ public void iCreateAnLdioJmsInComponent() { return Stream.of(toModel(content)); }; var ldioJmsInConfigurator = new LdioAmqpInAutoConfig().ldioConfigurator(ldioAmqpInRegistrator, null); - ldioJmsInConfigurator.configure(adapter, componentExecutor, properties); + ldioInput = ldioJmsInConfigurator.configure(adapter, componentExecutor, applicationEventPublisher, properties); } @And("^I send a model from (.*) and (.*) to broker using the amqp producer") @@ -91,9 +99,13 @@ public void iCreateDefaultConfigForLdioJmsInWithContentType(String contentType) config.put(OrchestratorConfig.ORCHESTRATOR_NAME, "orchestratorName"); } - @Then("Wait for the message") - public void theListenerWillWaitForTheMessage() { - await().until(() -> adapterResult.size() == 1); + @Then("Wait for {int} messages") + public void theListenerWillWaitForTheMessage(int i) { + await().until(() -> adapterResult.size() == i); + } + @Then("Wait for a grace period") + public void waitForGracePeriod() throws InterruptedException { + Awaitility.waitAtMost(Duration.of(500, ChronoUnit.MILLIS)); } @And("The result value will contain the model") @@ -103,9 +115,22 @@ public void theResultValueWillContainTheModel() { assertTrue(resultModel.isIsomorphicWith(inputModel)); } - @And("The componentExecutor will have been called") - public void theComponentExecutorWillHaveBeenCalled() { - assertEquals(1, componentExecutorResult.size()); + @When("I pause the pipeline") + public void pausePipeline() { + ldioInput.updateStatus(HALTED); + } + @When("I unpause the pipeline") + public void unPausePipeline() { + ldioInput.updateStatus(RESUMING); + } + @And("The result value will not contain the model") + public void theResultValueWillNotContainTheModel() { + assertEquals(0, adapterResult.size()); + } + + @And("The componentExecutor will have been called {int} times") + public void theComponentExecutorWillHaveBeenCalled(int interactions) { + assertEquals(interactions, componentExecutorResult.size()); } @ParameterType(value = "true|True|TRUE|false|False|FALSE") diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpIntegrationTest.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpIntegrationTest.java index d9492fdef..07d626d62 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpIntegrationTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpIntegrationTest.java @@ -9,6 +9,9 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.system.OutputCaptureExtension; import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationEventPublisher; + +import static org.mockito.Mockito.mock; @Suite @SpringBootTest @@ -20,7 +23,12 @@ public class AmqpIntegrationTest { @Autowired ApplicationContext applicationContext; + ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); + public LdioAmqpInRegistrator jmsInRegistrator() { return new LdioAmqpInRegistrator(applicationContext); } + public ApplicationEventPublisher applicationEventPublisher() { + return applicationEventPublisher; + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfigTest.java index af5d11646..26cef4ee6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfigTest.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; import java.util.HashMap; import java.util.Map; @@ -11,8 +12,10 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig.ORCHESTRATOR_NAME; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; class LdioAmqpInAutoConfigTest { + private ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); @Test void shouldThrowExceptionWhenInvalidUrlConfig() { var configurator = new LdioAmqpInAutoConfig.LdioJmsInConfigurator( @@ -23,7 +26,7 @@ void shouldThrowExceptionWhenInvalidUrlConfig() { ComponentProperties componentProperties = new ComponentProperties("pipelineName", NAME, config); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> configurator.configure((content) -> Stream.of(), null, componentProperties)); + () -> configurator.configure((content) -> Stream.of(), null, applicationEventPublisher, componentProperties)); assertEquals("Property remote-url is not in format of either " + "'amqp[s]://hostname:port[?option=value[&option2=value...]]' or " + @@ -38,7 +41,7 @@ void shouldNotThrowExceptionWhenNoUrlConfig() { Map config = getBasicConfig(); assertDoesNotThrow( - () -> configurator.configure((content) -> Stream.of(), null, new ComponentProperties("pipelineName", NAME, config))); + () -> configurator.configure((content) -> Stream.of(), null, applicationEventPublisher, new ComponentProperties("pipelineName", NAME, config))); } private Map getBasicConfig() { diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/resources/features/amqp-in-it.feature b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/resources/features/amqp-in-it.feature index 25f3b8532..b87aa1bf9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/resources/features/amqp-in-it.feature +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/resources/features/amqp-in-it.feature @@ -9,11 +9,32 @@ Feature: AMQPInIntegrationTest And I create default config for LdioJmsIn with And I start a listener with an LdioJmsIn component And I send a model from and to broker using the amqp producer - Then Wait for the message + Then Wait for 1 messages And The result value will contain the model - And The componentExecutor will have been called + And The componentExecutor will have been called 1 times + + Examples: + | queue | content-type | content | + | nquads | application/n-quads | 'my-zone-type' . | + | rdfxml | application/rdf+xml | my-zone-type | + + + Scenario Outline: Pausing the amqp input component + Given I create a queue for my scenario: + And I create a message producer + And I prepare the result lists + And I create default config for LdioJmsIn with + And I start a listener with an LdioJmsIn component + When I pause the pipeline + And I send a model from and to broker using the amqp producer + Then Wait for a grace period + And The result value will not contain the model + And The componentExecutor will have been called 0 times + When I unpause the pipeline + And I send a model from and to broker using the amqp producer + Then Wait for 2 messages + And The componentExecutor will have been called 2 times Examples: | queue | content-type | content | | nquads | application/n-quads | 'my-zone-type' . | - | rdfxml | application/rdf+xml | my-zone-type | \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java index edb9cb233..26c579c13 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java @@ -8,31 +8,56 @@ import org.apache.jena.riot.RDFParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; public class LdioArchiveFileIn extends LdioInput { public static final String NAME = "Ldio:ArchiveFileIn"; private final Logger log = LoggerFactory.getLogger(LdioArchiveFileIn.class); private final ArchiveFileCrawler archiveFileCrawler; private final Lang sourceFormat; + private boolean paused = false; - public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, ObservationRegistry observationRegistry, ArchiveFileCrawler crawler, Lang source) { - super(NAME, pipelineName, executor, null, observationRegistry); + public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher, ArchiveFileCrawler crawler, Lang source) { + super(NAME, pipelineName, executor, null, observationRegistry, applicationEventPublisher); this.archiveFileCrawler = crawler; this.sourceFormat = source; + updateStatus(STARTING); log.info("Starting with crawling the archive."); crawlArchive(); log.info("Finished crawling the archive."); } - public void crawlArchive() { + + @SuppressWarnings("java:S2273") + public synchronized void crawlArchive() { archiveFileCrawler.streamArchiveFilePaths().forEach(file -> { + while (paused) { + try { + this.wait(); + } catch (InterruptedException e) { + log.error("Thread interrupted: {}", e.getMessage()); + Thread.currentThread().interrupt(); + } + } Model model = RDFParser.source(file).lang(sourceFormat).toModel(); processModel(model); }); } @Override - public void shutdown() { - // Implemented in other story + public void shutdown(boolean keepState) { + this.paused = true; + } + @Override + protected synchronized void resume() { + this.paused = false; + this.notifyAll(); + } + + @Override + protected void pause() { + this.paused = true; } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java index ab6a353f3..681a5a436 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java @@ -2,14 +2,15 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; import be.vlaanderen.informatievlaanderen.ldes.ldio.ArchiveFileCrawler; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioArchiveFileIn; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -26,14 +27,15 @@ public class LdioArchiveFileInAutoConfig { public LdioInputConfigurator ldiArchiveFileInConfigurator(ObservationRegistry observationRegistry) { return new LdioInputConfigurator() { @Override - public LdiComponent configure(LdiAdapter adapter, - ComponentExecutor executor, - ComponentProperties config) { + public LdioInput configure(LdiAdapter adapter, + ComponentExecutor executor, + ApplicationEventPublisher applicationEventPublisher, + ComponentProperties config) { ArchiveFileCrawler archiveFileCrawler = new ArchiveFileCrawler(getArchiveDirectoryPath(config)); Lang hintLang = getSourceFormat(config); String pipelineName = config.getPipelineName(); - return new LdioArchiveFileIn(pipelineName, executor, observationRegistry, archiveFileCrawler, hintLang); + return new LdioArchiveFileIn(pipelineName, executor, observationRegistry, applicationEventPublisher, archiveFileCrawler, hintLang); } private Path getArchiveDirectoryPath(ComponentProperties config) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInIT.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInIT.java index f0ce10230..77a3f05fc 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInIT.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInIT.java @@ -3,10 +3,17 @@ import org.junit.platform.suite.api.IncludeEngines; import org.junit.platform.suite.api.SelectClasspathResource; import org.junit.platform.suite.api.Suite; +import org.springframework.context.ApplicationEventPublisher; + +import static org.mockito.Mockito.mock; @Suite @SuppressWarnings("java:S2187") @IncludeEngines("cucumber") @SelectClasspathResource("features") public class ArchiveFileInIT { + ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); + public ApplicationEventPublisher applicationEventPublisher() { + return applicationEventPublisher; + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java index f284a9eaa..aed80d3ff 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java @@ -8,6 +8,7 @@ import io.micrometer.observation.ObservationRegistry; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.RDFParser; +import org.springframework.context.ApplicationEventPublisher; import java.util.ArrayList; import java.util.List; @@ -19,7 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class ArchiveFileInITSteps { +public class ArchiveFileInITSteps extends ArchiveFileInIT{ + private final ApplicationEventPublisher applicationEventPublisher = applicationEventPublisher(); private List members; @@ -29,7 +31,7 @@ public void iCreateAnArchiveFileInComponentWithArchiveDir(String archiveDir) { ComponentExecutor componentExecutor = linkedDataModel -> members.add(linkedDataModel); var props = new ComponentProperties("pipeline", NAME, Map.of(ARCHIVE_ROOT_DIR_PROP, separatorsToSystem(archiveDir))); var ldioInputConfigurator = new LdioArchiveFileInAutoConfig().ldiArchiveFileInConfigurator(ObservationRegistry.create()); - ldioInputConfigurator.configure(null, componentExecutor, props); + ldioInputConfigurator.configure(null, componentExecutor, applicationEventPublisher, props); } @Then("All the members from the archive are passed to the pipeline in lexical order") diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java index e4e956531..2be2286cd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java @@ -4,13 +4,14 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.conversionstrategy.ConversionStrategy; import be.vlaanderen.informatievlaanderen.ldes.ldio.conversionstrategy.ConversionStrategyFactory; import be.vlaanderen.informatievlaanderen.ldes.ldio.util.MemberIdExtractor; -import com.azure.core.util.BinaryData; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.specialized.BlockBlobClient; import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.azure.core.util.BinaryData; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.specialized.BlockBlobClient; + @SuppressWarnings("java:S2629") public class LdiAzureBlobOut implements LdiOutput { public static final String NAME = "Ldio:AzureBlobOut"; diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java index 34458a627..c919dc74f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java @@ -14,11 +14,13 @@ import io.micrometer.observation.ObservationRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.http.HttpStatusCode; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import java.time.Instant; import java.util.List; +import java.util.concurrent.ScheduledFuture; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval.TYPE.CRON; @@ -30,10 +32,14 @@ public class LdioHttpInputPoller extends LdioInput implements Runnable { private final boolean continueOnFail; private static final Logger log = LoggerFactory.getLogger(LdioHttpInputPoller.class); private static final String CONTENT_TYPE = "Content-Type"; + private PollingInterval pollingInterval; + + @SuppressWarnings("java:S3740") + private ScheduledFuture scheduledPoll; public LdioHttpInputPoller(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, List endpoints, - boolean continueOnFail, RequestExecutor requestExecutor) { - super(NAME, pipelineName, executor, adapter, observationRegistry); + boolean continueOnFail, RequestExecutor requestExecutor, ApplicationEventPublisher applicationEventPublisher) { + super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); this.requestExecutor = requestExecutor; this.requests = endpoints.stream().map(endpoint -> new GetRequest(endpoint, RequestHeaders.empty())).toList(); this.continueOnFail = continueOnFail; @@ -43,11 +49,16 @@ public LdioHttpInputPoller(String pipelineName, ComponentExecutor executor, LdiA } public void schedulePoller(PollingInterval pollingInterval) { + this.pollingInterval = pollingInterval; scheduler.initialize(); + schedule(pollingInterval); + } + + private void schedule(PollingInterval pollingInterval) { if (pollingInterval.getType() == CRON) { - scheduler.schedule(this, pollingInterval.getCronTrigger()); + scheduledPoll = scheduler.schedule(this, pollingInterval.getCronTrigger()); } else { - scheduler.scheduleAtFixedRate(this, Instant.now(), pollingInterval.getDuration()); + scheduledPoll = scheduler.scheduleAtFixedRate(this, Instant.now(), pollingInterval.getDuration()); } } @@ -64,7 +75,7 @@ public void run() { }); } - public void shutdown() { + public void shutdown(boolean keepState) { this.scheduler.destroy(); } @@ -84,4 +95,16 @@ private void executeRequest(Request request) { throw new UnsuccesfulPollingException(response.getHttpStatus(), request.getUrl()); } } + + @Override + protected synchronized void resume() { + if (pollingInterval != null) { + schedule(pollingInterval); + } + } + + @Override + protected synchronized void pause() { + scheduledPoll.cancel(false); + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java index 726d5cc40..34f4e867c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java @@ -7,6 +7,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -15,6 +16,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInputPoller.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; @Configuration public class LdioHttpInputPollerAutoConfig { @@ -35,7 +37,7 @@ public HttpInputPollerConfigurator(ObservationRegistry observationRegistry) { @Override public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, - ComponentProperties properties) { + ApplicationEventPublisher applicationEventPublisher, ComponentProperties properties) { String pipelineName = properties.getPipelineName(); List endpoints = properties.getPropertyList(URL); @@ -43,10 +45,11 @@ public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor execu var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); - var httpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, observationRegistry, endpoints, continueOnFail, requestExecutor); + var httpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, observationRegistry, endpoints, continueOnFail, requestExecutor, applicationEventPublisher); httpInputPoller.schedulePoller(getPollingInterval(properties)); + httpInputPoller.updateStatus(STARTING); return httpInputPoller; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java index 2c211e1f4..c908ec67a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java @@ -6,9 +6,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.MissingHeaderException; -import com.github.tomakehurst.wiremock.client.CountMatchingStrategy; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -19,26 +16,37 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.scheduling.support.CronTrigger; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import com.github.tomakehurst.wiremock.client.CountMatchingStrategy; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + import static com.github.tomakehurst.wiremock.client.CountMatchingStrategy.GREATER_THAN_OR_EQUAL; -import static com.github.tomakehurst.wiremock.client.WireMock.reset; +import static com.github.tomakehurst.wiremock.client.CountMatchingStrategy.LESS_THAN_OR_EQUAL; import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.client.WireMock.reset; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.after; import static org.mockito.Mockito.*; +import static org.mockito.Mockito.after; @WireMockTest(httpPort = 10101) class LdioHttpInputPollerTest { private final LdiAdapter adapter = mock(LdiAdapter.class); private final ComponentExecutor executor = mock(ComponentExecutor.class); + @Autowired + ApplicationEventPublisher applicationEventPublisher; private final static String pipelineName = "pipeline"; private static final String BASE_URL = "http://localhost:10101"; private static final String ENDPOINT = "/resource"; @@ -57,12 +65,12 @@ void setUp() { .thenReturn(Stream.of()) .thenReturn(Stream.of()); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor); + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor, applicationEventPublisher); } @AfterEach void tearDown() { - ldioHttpInputPoller.shutdown(); + ldioHttpInputPoller.shutdown(false); } @Test @@ -78,7 +86,7 @@ void testClientPolling() { void whenPolling_andMissesHeader() { stubFor(get(ENDPOINT).willReturn(ok().withBody(CONTENT))); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor, applicationEventPublisher); Executable polling = () -> ldioHttpInputPoller.run(); assertThrows(MissingHeaderException.class, polling); @@ -96,6 +104,22 @@ void whenPeriodicPolling_thenReturnTwoTimesTheSameResponse(PollingInterval polli Mockito.verify(adapter, timeout(4000).times(2)).apply(LdiAdapter.Content.of(CONTENT, CONTENT_TYPE)); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), getRequestedFor(urlEqualTo(ENDPOINT))); } + @ParameterizedTest + @ArgumentsSource(PollingIntervalArgumentsProvider.class) + void when_PausePeriodicPolling_then_DontPoll(PollingInterval pollingInterval) { + stubFor(get(ENDPOINT).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); + + ldioHttpInputPoller.schedulePoller(pollingInterval); + ldioHttpInputPoller.pause(); + + await().atLeast(2, TimeUnit.SECONDS); + WireMock.verify(new CountMatchingStrategy(LESS_THAN_OR_EQUAL, 1), getRequestedFor(urlEqualTo(ENDPOINT))); + + ldioHttpInputPoller.resume(); + + Mockito.verify(adapter, timeout(4000).times(2)).apply(LdiAdapter.Content.of(CONTENT, CONTENT_TYPE)); + WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), getRequestedFor(urlEqualTo(ENDPOINT))); + } @Test void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldStillBePolled() { @@ -103,7 +127,7 @@ void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldSti String otherEndpoint = "/other-resource"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), - true, noAuthExecutor); + true, noAuthExecutor, applicationEventPublisher); ldioHttpInputPoller.run(); @@ -119,7 +143,7 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse(Poll String otherEndpoint = "/other-endpoint"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), - true, noAuthExecutor); + true, noAuthExecutor, applicationEventPublisher); ldioHttpInputPoller.schedulePoller(pollingInterval); @@ -146,7 +170,7 @@ void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling(PollingI void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(forbidden())); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor, applicationEventPublisher); ldioHttpInputPoller.schedulePoller(pollingInterval); Mockito.verify(adapter, after(4000).never()).apply(any()); @@ -157,7 +181,7 @@ void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling(Polling void when_EndpointDoesNotExist_Then_NoDataIsSent() { String wrongEndpoint = "/non-existing-resource"; ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + wrongEndpoint), true, - noAuthExecutor); + noAuthExecutor, applicationEventPublisher); ldioHttpInputPoller.run(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java index 5e60bc9e5..006df524b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java @@ -13,6 +13,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.MockedConstruction; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.scheduling.support.CronTrigger; import java.time.Duration; @@ -24,6 +26,8 @@ class LdioLdioHttpInputPollerAutoConfigTest { + @Autowired + ApplicationEventPublisher applicationEventPublisher; private final LdiAdapter adapter = mock(LdiAdapter.class); private final ComponentExecutor executor = mock(ComponentExecutor.class); private static final String BASE_URL = "http://localhost:10101"; @@ -54,7 +58,7 @@ void when_ValidIntervalConfig() { try (MockedConstruction ignored = mockConstruction(LdioHttpInputPoller.class)) { LdioHttpInputPoller poller = new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) - .configure(adapter, executor, createDefaultISOTestConfig()); + .configure(adapter, executor, applicationEventPublisher, createDefaultISOTestConfig()); verify(poller, times(1)).schedulePoller(new PollingInterval(Duration.of(1, ChronoUnit.SECONDS))); } } @@ -64,7 +68,7 @@ void when_ValidCronConfig() { try (MockedConstruction ignored = mockConstruction(LdioHttpInputPoller.class)) { LdioHttpInputPoller poller = new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) - .configure(adapter, executor, createDefaultCronTestConfig()); + .configure(adapter, executor, applicationEventPublisher, createDefaultCronTestConfig()); verify(poller, times(1)).schedulePoller(new PollingInterval(new CronTrigger("* * * * * *"))); } } @@ -74,7 +78,7 @@ void when_ValidCronConfig() { void whenInvalidIntervalConfigured_thenCatchException(String interval) { Executable configurePoller = () -> new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) - .configure(adapter, executor, createConfigWithInterval(BASE_URL + ENDPOINT, interval, "false")); + .configure(adapter, executor, applicationEventPublisher, createConfigWithInterval(BASE_URL + ENDPOINT, interval, "false")); assertThrows(IllegalArgumentException.class, configurePoller); } @@ -84,7 +88,7 @@ void whenInvalidIntervalConfigured_thenCatchException(String interval) { void whenInvalidCronConfigured_thenCatchException(String cron) { Executable configurePoller = () -> new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) - .configure(adapter, executor, createConfigWithCron(BASE_URL + ENDPOINT, cron, "false")); + .configure(adapter, executor, applicationEventPublisher, createConfigWithCron(BASE_URL + ENDPOINT, cron, "false")); assertThrows(IllegalArgumentException.class, configurePoller); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInController.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInController.java index 357ad3cf8..51b176991 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInController.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInController.java @@ -29,9 +29,15 @@ ResponseEntity processInput(@RequestHeader("Content-Type") String conten var contentType = contentTypeHeader.split(";")[0]; logIncomingRequest(contentType, contentLength, pipeline); - ofNullable(httpInProcesses.get(pipeline)) - .orElseThrow(() -> new PipelineDoesNotExistException(pipeline)) - .processInput(content, contentType); + + LdioHttpInProcess inputProcess = ofNullable(httpInProcesses.get(pipeline)) + .orElseThrow(() -> new PipelineDoesNotExistException(pipeline)); + if (inputProcess.isPaused()) { + return ResponseEntity.status(503).body(String.format("The LDIO pipeline named %s is currently paused.", pipeline)); + } else { + inputProcess.processInput(content, contentType); + } + return ResponseEntity.accepted().build(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java index 151dda798..2f2120042 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java @@ -4,16 +4,33 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import io.micrometer.observation.ObservationRegistry; +import org.springframework.context.ApplicationEventPublisher; public class LdioHttpInProcess extends LdioInput { public static final String NAME = "Ldio:HttpIn"; + private boolean isPaused = false; - public LdioHttpInProcess(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry) { - super(NAME, pipelineName, executor, adapter, observationRegistry); + public LdioHttpInProcess(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, + ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher) { + super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); } @Override - public void shutdown() { + protected void resume() { + this.isPaused = false; + } + + @Override + protected void pause() { + this.isPaused = true; + } + + public boolean isPaused() { + return isPaused; + } + + @Override + public void shutdown(boolean keepState) { // Not implementable for push based } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java index b41af23d4..43a9bea07 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java @@ -5,6 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInProcess; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.HttpInPipelineCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.springframework.context.ApplicationEventPublisher; @@ -13,6 +14,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInProcess.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.exception.LdiAdapterMissingException.verifyAdapterPresent; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; @Configuration public class LdioHttpInAutoConfig { @@ -35,14 +37,16 @@ public LdioHttpInConfigurator(ApplicationEventPublisher eventPublisher, Observat } @Override - public Object configure(LdiAdapter adapter, - ComponentExecutor executor, - ComponentProperties config) { + public LdioInput configure(LdiAdapter adapter, + ComponentExecutor executor, + ApplicationEventPublisher applicationEventPublisher, + ComponentProperties config) { String pipelineName = config.getPipelineName(); verifyAdapterPresent(pipelineName, adapter); - LdioHttpInProcess ldioHttpIn = new LdioHttpInProcess(pipelineName, executor, adapter, observationRegistry); + LdioHttpInProcess ldioHttpIn = new LdioHttpInProcess(pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); + ldioHttpIn.updateStatus(STARTING); eventPublisher.publishEvent(new HttpInPipelineCreatedEvent(pipelineName, ldioHttpIn)); return ldioHttpIn; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java index cc375bdb9..ce82f5cbe 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInAutoConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -16,9 +17,10 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInProcess.NAME; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RESUMING; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -27,10 +29,11 @@ class LdioHttpInputTest { private final String endpoint = "endpoint"; @Autowired - ApplicationEventPublisher eventPublisher; + private ApplicationEventPublisher eventPublisher; @Autowired private MockMvc mockMvc; private LdiAdapter adapter; + private LdioInput input; @BeforeEach void setup() { @@ -39,14 +42,31 @@ void setup() { when(adapter.apply(any())).thenReturn(Stream.empty()); - new LdioHttpInAutoConfig.LdioHttpInConfigurator(eventPublisher, null) - .configure(adapter, executor, new ComponentProperties(endpoint, NAME)); + input = (LdioInput) new LdioHttpInAutoConfig.LdioHttpInConfigurator(eventPublisher, null) + .configure(adapter, executor, eventPublisher, new ComponentProperties(endpoint, NAME)); } @Test void testHttpEndpoint() throws Exception { String content = "_:b0 \"Jane Doe\" ."; String contentType = "application/n-quads"; + input.updateStatus(RESUMING); + + mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().isAccepted()); + + verify(adapter).apply(LdiAdapter.Content.of(content, contentType)); + } + @Test + void when_PipelineIsHalted_Then_MessageIsNotProcessed() throws Exception { + String content = "_:b0 \"Jane Doe\" ."; + String contentType = "application/n-quads"; + input.updateStatus(HALTED); + + mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().is(503)); + + verifyNoInteractions(adapter); + + input.updateStatus(RESUMING); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().isAccepted()); diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index c40a52087..0a0acf13b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -26,5 +26,10 @@ spring-kafka-test test + + org.springframework.boot + spring-boot-test + test + diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java index c742152ce..4f62200d2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java @@ -2,50 +2,106 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import be.vlaanderen.informatievlaanderen.ldes.ldio.auth.KafkaAuthStrategy; +import be.vlaanderen.informatievlaanderen.ldes.ldio.auth.SaslSslPlainConfigProvider; +import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.SecurityProtocolNotSupportedException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.listener.LdioKafkaInListener; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.kafka.listener.MessageListener; - -public class LdioKafkaIn extends LdioInput implements MessageListener { - public static final String NAME = "Ldio:KafkaIn"; - private static final Logger log = LoggerFactory.getLogger(LdioKafkaIn.class); - private final String defaultContentType; - - /** - * Creates a Kafka Listener with its Component Executor and LDI Adapter - * - * @param componentName References the Fully Qualified name of the LDIO component - * @param pipelineName Name of the LDIO pipeline to which the LDI Input belongs - * @param executor Instance of the Component Executor. Allows the LDI Input to pass - * data on the pipeline - * @param adapter Instance of the LDI Adapter. Facilitates transforming the input - * data to a linked data model (RDF). - */ - public LdioKafkaIn(String componentName, String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, String defaultContentType) { - super(componentName, pipelineName, executor, adapter, observationRegistry); - this.defaultContentType = defaultContentType; - } +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.KafkaMessageListenerContainer; - @Override - public void onMessage(ConsumerRecord data) { - final String contentType = determineContentType(data.headers()); - final var content = LdiAdapter.Content.of(data.value(), contentType); - log.atDebug().log("Incoming kafka message: {}", content); - processInput(content); - } +import java.util.HashMap; +import java.util.Map; - private String determineContentType(Headers headers) { - final Header contentTypeOnHeader = headers.lastHeader("content-type"); - return contentTypeOnHeader == null ? defaultContentType : new String(contentTypeOnHeader.value()); - } +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.KafkaInConfigKeys.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.KafkaInConfigKeys.SASL_JAAS_PASSWORD; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig.ORCHESTRATOR_NAME; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; + +public class LdioKafkaIn extends LdioInput { + public static final String NAME = "Ldio:KafkaIn"; + @SuppressWarnings("java:S3740") + private final KafkaMessageListenerContainer container; + private final LdioKafkaInListener listener; + /** + * Creates a LdiInput with its Component Executor and LDI Adapter + * + * @param pipelineName + * @param executor Instance of the Component Executor. Allows the LDI Input to pass + * data on the pipeline + * @param adapter Instance of the LDI Adapter. Facilitates transforming the input + * data to a linked data model (RDF). + * @param observationRegistry + * @param applicationEventPublisher + * @param config + */ + public LdioKafkaIn(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, + ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { + super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); + this.listener = new LdioKafkaInListener(getContentType(config), this::processInput); + var consumerFactory = new DefaultKafkaConsumerFactory<>(getConsumerConfig(config)); + ContainerProperties containerProps = new ContainerProperties(config.getProperty(TOPICS).split(",")); + containerProps.setMessageListener(listener); + this.container = new KafkaMessageListenerContainer<>(consumerFactory, containerProps); + container.start(); + this.updateStatus(STARTING); + } + + @Override + protected void resume() { + container.resume(); + } + + @Override + protected void pause() { + container.pause(); + } + + private String getContentType(ComponentProperties config) { + return config + .getOptionalProperty(CONTENT_TYPE) + .map(RDFLanguages::contentTypeToLang) + .map(Lang::getHeaderString) + .orElse(Lang.NQUADS.getHeaderString()); + } @Override - public void shutdown() { - // implemented in status story + public void shutdown(boolean keepState) { + container.pause(); } + private Map getConsumerConfig(ComponentProperties config) { + final Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getProperty(BOOTSTRAP_SERVERS)); + props.put(ConsumerConfig.GROUP_ID_CONFIG, + config.getOptionalProperty(GROUP_ID).orElse(defineUniqueGroupName(config))); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + config.getOptionalProperty(AUTO_OFFSET_RESET).orElse("earliest")); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + + var authStrategy = KafkaAuthStrategy.from(config.getOptionalProperty(SECURITY_PROTOCOL) + .orElse(KafkaAuthStrategy.NO_AUTH.name())) + .orElseThrow(() -> new SecurityProtocolNotSupportedException(SECURITY_PROTOCOL)); + + if (KafkaAuthStrategy.SASL_SSL_PLAIN.equals(authStrategy)) { + final String user = config.getProperty(SASL_JAAS_USER); + final String password = config.getProperty(SASL_JAAS_PASSWORD); + props.putAll(new SaslSslPlainConfigProvider().createSaslSslPlainConfig(user, password)); + } + + return props; + } + + private String defineUniqueGroupName(ComponentProperties config) { + return String.format("ldio-%s-%s", config.getProperty(ORCHESTRATOR_NAME), + config.getPipelineName()); + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java index f8803396d..80c20be65 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java @@ -3,28 +3,15 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn; -import be.vlaanderen.informatievlaanderen.ldes.ldio.auth.KafkaAuthStrategy; -import be.vlaanderen.informatievlaanderen.ldes.ldio.auth.SaslSslPlainConfigProvider; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; -import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.SecurityProtocolNotSupportedException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFLanguages; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.listener.ContainerProperties; -import org.springframework.kafka.listener.KafkaMessageListenerContainer; - -import java.util.HashMap; -import java.util.Map; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.KafkaInConfigKeys.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig.ORCHESTRATOR_NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.exception.LdiAdapterMissingException.verifyAdapterPresent; @Configuration @@ -44,50 +31,11 @@ public LdioKafkaInConfigurator(ObservationRegistry observationRegistry) { } @Override - public Object configure(LdiAdapter adapter, ComponentExecutor executor, ComponentProperties config) { + public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { String pipelineName = config.getPipelineName(); verifyAdapterPresent(pipelineName, adapter); - LdioKafkaIn ldioKafkaIn = new LdioKafkaIn(NAME, pipelineName, executor, adapter, observationRegistry, getContentType(config)); - var consumerFactory = new DefaultKafkaConsumerFactory<>(getConsumerConfig(config)); - ContainerProperties containerProps = new ContainerProperties(config.getProperty(TOPICS).split(",")); - containerProps.setMessageListener(ldioKafkaIn); - return new KafkaMessageListenerContainer<>(consumerFactory, containerProps); - } - - private String getContentType(ComponentProperties config) { - return config - .getOptionalProperty(CONTENT_TYPE) - .map(RDFLanguages::contentTypeToLang) - .map(Lang::getHeaderString) - .orElse(Lang.NQUADS.getHeaderString()); - } - - private Map getConsumerConfig(ComponentProperties config) { - final Map props = new HashMap<>(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getProperty(BOOTSTRAP_SERVERS)); - props.put(ConsumerConfig.GROUP_ID_CONFIG, - config.getOptionalProperty(GROUP_ID).orElse(defineUniqueGroupName(config))); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - config.getOptionalProperty(AUTO_OFFSET_RESET).orElse("earliest")); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - - var authStrategy = KafkaAuthStrategy.from(config.getOptionalProperty(SECURITY_PROTOCOL) - .orElse(KafkaAuthStrategy.NO_AUTH.name())) - .orElseThrow(() -> new SecurityProtocolNotSupportedException(SECURITY_PROTOCOL)); - - if (KafkaAuthStrategy.SASL_SSL_PLAIN.equals(authStrategy)) { - final String user = config.getProperty(SASL_JAAS_USER); - final String password = config.getProperty(SASL_JAAS_PASSWORD); - props.putAll(new SaslSslPlainConfigProvider().createSaslSslPlainConfig(user, password)); - } - - return props; - } - - private String defineUniqueGroupName(ComponentProperties config) { - return String.format("ldio-%s-%s", config.getProperty(ORCHESTRATOR_NAME), config.getPipelineName()); + return new LdioKafkaIn(pipelineName, executor, adapter, observationRegistry, applicationEventPublisher, config); } } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/listener/LdioKafkaInListener.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/listener/LdioKafkaInListener.java new file mode 100644 index 000000000..874cd4214 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/listener/LdioKafkaInListener.java @@ -0,0 +1,35 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.listener; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.kafka.listener.MessageListener; + +import java.util.function.Consumer; + +public class LdioKafkaInListener implements MessageListener { + private static final Logger log = LoggerFactory.getLogger(LdioKafkaInListener.class); + private final Consumer messageCallback; + private final String defaultContentType; + + public LdioKafkaInListener(String defaultContentType, Consumer callback) { + this.defaultContentType = defaultContentType; + this.messageCallback = callback; + } + + @Override + public void onMessage(ConsumerRecord data) { + final String contentType = determineContentType(data.headers()); + final var content = LdiAdapter.Content.of(data.value(), contentType); + log.atDebug().log("Incoming kafka message: {}", content); + messageCallback.accept(content); + } + + private String determineContentType(Headers headers) { + final Header contentTypeOnHeader = headers.lastHeader("content-type"); + return contentTypeOnHeader == null ? defaultContentType : new String(contentTypeOnHeader.value()); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaCommonIntegrationSteps.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaCommonIntegrationSteps.java index 395f567d8..eecf412ff 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaCommonIntegrationSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaCommonIntegrationSteps.java @@ -3,7 +3,7 @@ import io.cucumber.java.en.Given; import org.springframework.kafka.test.EmbeddedKafkaBroker; -public class KafkaCommonIntegrationSteps { +public class KafkaCommonIntegrationSteps extends KafkaIntegrationTest { public static String topic; public static final String bootstrapServer = "localhost:9095"; private static final EmbeddedKafkaBroker embeddedKafkaBroker; diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java index df9dd9e21..55583cd3c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java @@ -2,9 +2,12 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.RDFParser; import org.apache.jena.riot.RDFParserBuilder; @@ -13,14 +16,16 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.serialization.StringSerializer; +import org.awaitility.Awaitility; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.listener.KafkaMessageListenerContainer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn.NAME; @@ -33,7 +38,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class KafkaInIntegrationTestSteps { +public class KafkaInIntegrationTestSteps extends KafkaIntegrationTest { + private final ApplicationEventPublisher applicationEventPublisher = applicationEventPublisher(); private Model inputModel; private String contentType; @@ -41,6 +47,7 @@ public class KafkaInIntegrationTestSteps { private List adapterResult; private List componentExecutorResult; private KafkaTemplate kafkaProducer; + private LdioInput kafkaIn; @And("I prepare the result lists") public void iPrepareTheResultLists() { @@ -58,9 +65,7 @@ public void iCreateAnLdioKafkaInComponent() { return Stream.of(toModel(content)); }; var ldioKafkaInConfigurator = new LdioKafkaInAutoConfig().ldioConfigurator(null); - var ldioKafkaInContainer = (KafkaMessageListenerContainer) ldioKafkaInConfigurator - .configure(adapter, componentExecutor, properties); - ldioKafkaInContainer.start(); + kafkaIn = ldioKafkaInConfigurator.configure(adapter, componentExecutor, applicationEventPublisher, properties); } private Model toModel(LdiAdapter.Content content) { @@ -105,6 +110,10 @@ public void iSendTheModelToTheUsingTheKafkaProducer() { public void theListenerWillWaitForTheMessage() { await().until(() -> adapterResult.size() == 1); } + @Then("Wait for a grace period") + public void theListenerWillWaitForPeriod() throws InterruptedException { + Awaitility.waitAtMost(1500, TimeUnit.MILLISECONDS); + } @And("^The result header will contain the (.*)$") public void theResultHeaderWillContainTheContentType(String expectedContentType) { @@ -118,8 +127,18 @@ public void theResultValueWillContainTheModel() { assertTrue(resultModel.isIsomorphicWith(inputModel)); } - @And("The componentExecutor will have been called") - public void theComponentExecutorWillHaveBeenCalled() { - assertEquals(1, componentExecutorResult.size()); + @And("The componentExecutor will have been called {int} times") + public void theComponentExecutorWillHaveBeenCalled(int i) { + assertEquals(i, componentExecutorResult.size()); + } + @When("I pause the pipeline") + public void pauseInput() { + kafkaIn.updateStatus(PipelineStatus.HALTED); + } + @When("I unpause the pipeline") + public void unPauseInput() { + kafkaIn.updateStatus(PipelineStatus.RESUMING); } } + + diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaIntegrationTest.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaIntegrationTest.java index 4da204683..112554883 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaIntegrationTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaIntegrationTest.java @@ -3,10 +3,20 @@ import org.junit.platform.suite.api.IncludeEngines; import org.junit.platform.suite.api.SelectClasspathResource; import org.junit.platform.suite.api.Suite; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationEventPublisher; + +import static org.mockito.Mockito.mock; @Suite @SuppressWarnings("java:S2187") @IncludeEngines("cucumber") +@SpringBootTest @SelectClasspathResource("features") public class KafkaIntegrationTest { + + ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); + public ApplicationEventPublisher applicationEventPublisher() { + return applicationEventPublisher; + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaOutIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaOutIntegrationTestSteps.java index 40bb37e7a..612b63c1f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaOutIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaOutIntegrationTestSteps.java @@ -30,7 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class KafkaOutIntegrationTestSteps { +public class KafkaOutIntegrationTestSteps extends KafkaIntegrationTest { private LdiOutput ldioKafkaOut; private Model inputModel; diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java index 55cca11f5..6a0995eca 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java @@ -6,6 +6,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.kafka.test.rule.EmbeddedKafkaRule; import java.util.HashMap; import java.util.Map; @@ -14,9 +16,14 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig.ORCHESTRATOR_NAME; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; class LdioKafkaInAutoConfigTest { + private ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); + private static final String TOPIC = "TopicName"; + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, TOPIC); + @Test void shouldThrowExceptionWhenInvalidAuthConfig() { var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(ObservationRegistry.create()); @@ -26,7 +33,7 @@ void shouldThrowExceptionWhenInvalidAuthConfig() { ComponentProperties componentProperties = new ComponentProperties("pipelineName", NAME, config); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> configurator.configure((content) -> Stream.of(), null, componentProperties)); + () -> configurator.configure((content) -> Stream.of(), null, applicationEventPublisher, componentProperties)); assertEquals("java.lang.IllegalArgumentException: Invalid 'security-protocol', " + "the supported protocols are: [NO_AUTH, SASL_SSL_PLAIN]", exception.getMessage()); @@ -39,7 +46,7 @@ void shouldNotThrowExceptionWhenNoAuthConfig() { Map config = getBasicConfig(); assertDoesNotThrow( - () -> configurator.configure((content) -> Stream.of(), null, new ComponentProperties("pipelineName", NAME, config))); + () -> configurator.configure((content) -> Stream.of(), null, applicationEventPublisher, new ComponentProperties("pipelineName", NAME, config))); } @Test @@ -52,15 +59,15 @@ void shouldNotThrowExceptionWhenSaslSslPlain() { config.put(KafkaInConfigKeys.SASL_JAAS_PASSWORD, "secret"); assertDoesNotThrow( - () -> configurator.configure((content) -> Stream.of(), null, new ComponentProperties("pipelineName", NAME, config))); + () -> configurator.configure((content) -> Stream.of(), null, applicationEventPublisher, new ComponentProperties("pipelineName", NAME, config))); } private Map getBasicConfig() { Map config = new HashMap<>(); - config.put(KafkaInConfigKeys.BOOTSTRAP_SERVERS, "servers"); - config.put(KafkaInConfigKeys.TOPICS, "topic1"); + config.put(KafkaInConfigKeys.BOOTSTRAP_SERVERS, embeddedKafka.getEmbeddedKafka().getBrokersAsString()); + config.put(KafkaInConfigKeys.TOPICS, TOPIC); config.put(ORCHESTRATOR_NAME, "orchestrator.name"); return config; } -} \ No newline at end of file +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/resources/features/kafka-in-it.feature b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/resources/features/kafka-in-it.feature index 4ef66e78b..a8e7e68ea 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/resources/features/kafka-in-it.feature +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/resources/features/kafka-in-it.feature @@ -13,9 +13,30 @@ Feature: KafkaInIntegrationTest Then Wait for the message And The result header will contain the And The result value will contain the model - And The componentExecutor will have been called + And The componentExecutor will have been called 1 times Examples: | topic | content-type | content | | nquads | application/n-quads | 'my-zone-type' . | - | rdfxml | application/rdf+xml | my-zone-type | \ No newline at end of file + | rdfxml | application/rdf+xml | my-zone-type | + + Scenario Outline: Pausing an ldioKafkaIn topic + Given I create a topic for my scenario: + And I prepare the result lists + And I create default config for LdioKafkaIn + And I start a listener with an LdioKafkaIn component + And I create a kafka producer + And I create a model from and + When I pause the pipeline + And I send the model to broker using the kafka producer + Then Wait for a grace period + And The componentExecutor will have been called 0 times + When I unpause the pipeline + Then Wait for the message + And The result header will contain the + And The result value will contain the model + And The componentExecutor will have been called 1 times + + Examples: + | topic | content-type | content | + | test | application/n-quads | 'my-zone-type' . | \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java index e64601b0b..d89ccbaeb 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java @@ -3,14 +3,18 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TransferService; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; + public class LdioLdesClientConnectorApi { private final TransferService transferService; private final TokenService tokenService; + private final LdioLdesClient ldesClient; @SuppressWarnings("java:S107") - public LdioLdesClientConnectorApi(TransferService transferService, TokenService tokenService) { + public LdioLdesClientConnectorApi(TransferService transferService, TokenService tokenService, LdioLdesClient ldesClient) { this.transferService = transferService; this.tokenService = tokenService; + this.ldesClient = ldesClient; } public void handleToken(String token) { @@ -18,7 +22,9 @@ public void handleToken(String token) { } public String handleTransfer(String transfer) { - return transferService.startTransfer(transfer).getBodyAsString() + String transer = transferService.startTransfer(transfer).getBodyAsString() .orElse(""); + ldesClient.updateStatus(STARTING); + return transer; } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 323fd192d..4185ff99a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -11,6 +11,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.LdesClientConnectorApiCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.MemberSupplier; @@ -46,7 +47,7 @@ public LdioClientConnectorConfigurator(ApplicationEventPublisher eventPublisher, } @Override - public Object configure(LdiAdapter adapter, ComponentExecutor executor, ComponentProperties properties) { + public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties properties) { final String pipelineName = properties.getPipelineName(); final var connectorTransferUrl = properties.getProperty(CONNECTOR_TRANSFER_URL); final var transferService = new MemoryTransferService(baseRequestExecutor, connectorTransferUrl); @@ -56,9 +57,9 @@ public Object configure(LdiAdapter adapter, ComponentExecutor executor, Componen final var edcRequestExecutor = requestExecutorFactory.createEdcExecutor(baseRequestExecutor, tokenService, urlProxy); final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, edcRequestExecutor).getMemberSupplier(); - var ldesClient = new LdioLdesClient(pipelineName, executor, observationRegistry, memberSupplier); + var ldesClient = new LdioLdesClient(pipelineName, executor, observationRegistry, memberSupplier, applicationEventPublisher); ldesClient.start(); - eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService))); + eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java index e3d52bbe8..63742ba22 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java @@ -32,13 +32,15 @@ class LdioLdesClientConnectorTest { private final String endpoint = "endpoint"; private TransferService transferService; private TokenService tokenService; + private LdioLdesClient ldesClient; @BeforeEach void setup() { transferService = mock(TransferService.class); tokenService = mock(TokenService.class); + ldesClient = mock(LdioLdesClient.class); - eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(endpoint, new LdioLdesClientConnectorApi(transferService, tokenService))); + eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(endpoint, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); } @Test diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 183381a8f..aff914f79 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -2,11 +2,13 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.MemberSupplier; import ldes.client.treenodesupplier.domain.valueobject.EndOfLdesException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; import java.util.concurrent.ExecutorService; @@ -20,24 +22,29 @@ public class LdioLdesClient extends LdioInput { private final MemberSupplier memberSupplier; private boolean threadRunning = true; + private boolean paused = false; public LdioLdesClient(String pipelineName, ComponentExecutor componentExecutor, ObservationRegistry observationRegistry, - MemberSupplier memberSupplier) { - super(NAME, pipelineName, componentExecutor, null, observationRegistry); + MemberSupplier memberSupplier, + ApplicationEventPublisher applicationEventPublisher) { + super(NAME, pipelineName, componentExecutor, null, observationRegistry, applicationEventPublisher); this.memberSupplier = memberSupplier; } @SuppressWarnings("java:S2095") public void start() { + memberSupplier.init(); + this.updateStatus(PipelineStatus.STARTING); final ExecutorService executorService = newSingleThreadExecutor(); executorService.submit(this::run); } - private void run() { + private synchronized void run() { try { while (threadRunning) { + checkPause(); processModel(memberSupplier.get().getModel()); } } catch (EndOfLdesException e) { @@ -47,8 +54,33 @@ private void run() { } } + private synchronized void checkPause() { + while (paused) { + try { + this.wait(); + } catch (InterruptedException e) { + log.error("Thread interrupted: {}", e.getMessage()); + Thread.currentThread().interrupt(); + } + } + } + @Override - public void shutdown() { + public void shutdown(boolean keepState) { threadRunning = false; + if (!keepState) { + memberSupplier.destroyState(); + } + } + + @Override + protected synchronized void resume() { + this.paused = false; + this.notifyAll(); + } + + @Override + protected void pause() { + this.paused = true; } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index 0fadaec09..36beef476 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -3,13 +3,14 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.MemberSupplier; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -30,12 +31,13 @@ public LdioLdesClientConfigurator(ObservationRegistry observationRegistry) { } @Override - public LdiComponent configure(LdiAdapter adapter, ComponentExecutor componentExecutor, - ComponentProperties properties) { + public LdioInput configure(LdiAdapter adapter, ComponentExecutor componentExecutor, + ApplicationEventPublisher applicationEventPublisher, + ComponentProperties properties) { String pipelineName = properties.getPipelineName(); RequestExecutor requestExecutor = new LdioRequestExecutorSupplier().getRequestExecutor(properties); final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, requestExecutor).getMemberSupplier(); - var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, observationRegistry, memberSupplier); + var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, observationRegistry, memberSupplier, applicationEventPublisher); ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientInIT.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientInIT.java index b96dd13dd..d71377ae4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientInIT.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientInIT.java @@ -3,6 +3,8 @@ import org.junit.platform.suite.api.IncludeEngines; import org.junit.platform.suite.api.SelectClasspathResource; import org.junit.platform.suite.api.Suite; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import com.github.tomakehurst.wiremock.junit5.WireMockTest; @@ -12,4 +14,9 @@ @SelectClasspathResource("features") @WireMockTest(httpPort = 10101) public class LdesClientInIT { + @Autowired + ApplicationEventPublisher applicationEventPublisher; + public ApplicationEventPublisher applicationEventPublisher() { + return applicationEventPublisher; + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java index 7db518b1d..17fc78f81 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java @@ -3,13 +3,13 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import com.github.tomakehurst.wiremock.WireMockServer; import io.cucumber.java.BeforeAll; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; import org.apache.jena.rdf.model.Model; +import org.springframework.context.ApplicationEventPublisher; import java.time.Duration; import java.util.ArrayList; @@ -18,15 +18,18 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import com.github.tomakehurst.wiremock.WireMockServer; + import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.URLS; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static org.awaitility.Awaitility.await; -public class LdioLdesClientITSteps { +public class LdioLdesClientITSteps extends LdesClientInIT{ private List members; static WireMockServer wireMockServer = new WireMockServer(options().port(10101)); private final Map componentPropsMap = new HashMap<>(); + private final ApplicationEventPublisher applicationEventPublisher = applicationEventPublisher(); @BeforeAll public static void before_all() { @@ -55,7 +58,7 @@ public void iStartAnLdesLdioInComponentWithUrl() { var props = new ComponentProperties("pipelineName", NAME, componentPropsMap); var ldioInputConfigurator = new LdioLdesClientAutoConfig().ldioConfigurator(null); - ldioInputConfigurator.configure(null, componentExecutor, props); + ldioInputConfigurator.configure(null, componentExecutor, applicationEventPublisher, props); } @Then("All {int} members from the stream are passed to the pipeline")