From 18c2587fcac98f78a02eca3405528279d945fa78 Mon Sep 17 00:00:00 2001 From: Yalz Date: Tue, 12 Dec 2023 12:22:36 +0100 Subject: [PATCH 1/5] fix: fix RmlAdapter incompatibility with WebFlux (HttpIn) --- .../vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java b/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java index 9e1f15b01..296f896c9 100644 --- a/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java +++ b/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java @@ -21,6 +21,7 @@ import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.ValueFactory; import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.util.ModelCollector; import org.eclipse.rdf4j.model.vocabulary.RDF; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.Rio; @@ -53,7 +54,8 @@ public RmlAdapter(String mappingString) { @Override public Stream apply(Content content) { - Dataset dataset = rmlMapper.mapToModel(IOUtils.toInputStream(content.content(), StandardCharsets.UTF_8)) + Dataset dataset = Objects.requireNonNull(rmlMapper.map(IOUtils.toInputStream(content.content(), StandardCharsets.UTF_8)) + .collect(ModelCollector.toModel()).share().block()) .stream().map(JenaConverters::toQuad) .collect(JenaCollectors.toDataset()); From a1b34b4facf927e1728b166755e60557e90a1bfc Mon Sep 17 00:00:00 2001 From: jonasbul Date: Wed, 13 Dec 2023 19:32:40 +0100 Subject: [PATCH 2/5] ci: prepare 1.12.0 release --- ldi-api/pom.xml | 2 +- ldi-core/file-archiving/pom.xml | 2 +- ldi-core/geojson-to-wkt/pom.xml | 2 +- ldi-core/json-to-ld-adapter/pom.xml | 2 +- ldi-core/ldes-client/pom.xml | 2 +- ldi-core/ldes-client/starting-node/pom.xml | 2 +- ldi-core/ldes-client/tree-node-fetcher/pom.xml | 2 +- ldi-core/ldes-client/tree-node-supplier/pom.xml | 2 +- ldi-core/ldi-common/pom.xml | 2 +- ldi-core/ngsiv2-to-ld-adapter/pom.xml | 2 +- ldi-core/pom.xml | 2 +- ldi-core/rdf-adapter/pom.xml | 2 +- ldi-core/repository-materialiser/pom.xml | 2 +- ldi-core/request-executor/pom.xml | 2 +- ldi-core/rml-adapter/pom.xml | 2 +- ldi-core/sparql-construct/pom.xml | 2 +- ldi-core/version-materialiser/pom.xml | 2 +- ldi-core/version-object-creator/pom.xml | 2 +- ldi-nifi/ldi-nifi-common/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml | 2 +- .../create-version-object-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/pom.xml | 2 +- .../rdf4j-repository-materialisation-processor/pom.xml | 2 +- .../ldi-nifi-processors/sparql-interactions-processor/pom.xml | 2 +- .../version-materialisation-processor/pom.xml | 2 +- ldi-nifi/pom.xml | 2 +- ldi-orchestrator/ldio-application/pom.xml | 2 +- ldi-orchestrator/ldio-common/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml | 2 +- .../ldio-connectors/ldio-json-to-ld-adapter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml | 2 +- .../ldio-connectors/ldio-ldes-client-connector/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml | 2 +- .../ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml | 4 ++-- ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml | 2 +- .../ldio-connectors/ldio-repository-materialiser/pom.xml | 2 +- .../ldio-connectors/ldio-request-executor/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml | 2 +- .../ldio-connectors/ldio-sparql-construct/pom.xml | 2 +- .../ldio-connectors/ldio-version-materialiser/pom.xml | 2 +- .../ldio-connectors/ldio-version-object-creator/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/pom.xml | 2 +- ldi-orchestrator/pom.xml | 2 +- pom.xml | 2 +- 57 files changed, 58 insertions(+), 58 deletions(-) diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index b01a93d01..cc1f61dd1 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index 81672ffa8..b383a7914 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index afd04590f..6f3398249 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index dad460b55..08b79876d 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index cbf70b569..c2a8146f0 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 2ceef28ea..497a6fbf4 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index f145eb548..64658ee70 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 tree-node-fetcher diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index 6e6541a79..30f3f6810 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 tree-node-supplier diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index 80d2e0c9b..6c7b09b0e 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldi-common diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index aaf8f3715..6bb28d9b5 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index a687318fc..43821f05e 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index bbf39e090..8ae40ef1f 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index edaa7d051..0371a4506 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index cedad9c63..40e4fed12 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index 7d1057f28..4178792b5 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index 5d4635b2a..6be273dc7 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index 7fc1758a3..c394ae258 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index 4048046a4..db0f234c2 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index dc3b70a03..966428343 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index 71ab1c074..a7d44d58d 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index d35bfea80..82a461feb 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index bc5542629..649dab596 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index 4c62b8845..fe0f1138d 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index f730b990d..f347f3f30 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 4153f9cd2..168d71fef 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index ef27214bf..24b8db746 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 5d092b155..67f842e84 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml index 3c78f85ad..dfd880be9 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT rdf4j-repository-materialisation-processor diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index 8d5ada8da..908050886 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index c7eca674d..c4b6b145b 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index da80f8af0..965393c11 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index aecc9312c..bef3e998e 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index ddf85502f..293e92929 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index 1f28eff03..120196fd5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index 5ecefd964..a527e00f0 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 4896bfd1b..bcca193c1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index 355e9c7a4..baa8b6664 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index c78908f6f..84b58b692 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index f8f63bd44..41bed65dd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index bd83c7b51..c0200cce6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 45b02184a..97b7421bd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index 8baa22639..85b58026f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index 2e66421fc..5b77d4d9b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 0f5258556..fc1b9e7a7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index 09c0bd423..b8f58c8ea 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 3b66ee89e..2f78111c0 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index 90fd93ae3..c6dcc465b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter @@ -15,7 +15,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ngsiv2-to-ld-adapter - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index e8467c97e..8a860f3ab 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml index 7e77af580..738fe6829 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-repository-materialiser diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index 88e2e2764..2549c0567 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index 65de34550..5a9cb7966 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index 7a8e416c8..8268035f8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index 425bb40f5..0d9827ca4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index 3f437ea4b..cc60f7a25 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index d8030306f..9985299df 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 588b0c4fc..d9d88f0cd 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 799b6dbda..9acadb3b2 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 1.11.0-SNAPSHOT + 1.12.0-SNAPSHOT ldi-api From 4b364762826b11737f4bc4c0908b5defbfdd1209 Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:04:39 +0100 Subject: [PATCH 3/5] fix: prevent ldesClient from blocking ldio flow with infinite loop (#433) --- ...r.java => LdioLdesClientConnectorApi.java} | 16 ++++-------- .../LdioLdesClientConnectorAutoConfig.java | 15 ++++++----- .../ldio/LdioLdesClientConnectorTest.java | 4 +-- .../ldes/ldio/LdioLdesClient.java | 26 ++++++++++--------- .../ldio/config/LdioLdesClientAutoConfig.java | 6 ++--- .../resources/features/ldes-client-in.feature | 2 +- 6 files changed, 34 insertions(+), 35 deletions(-) rename ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/{LdioLdesClientConnector.java => LdioLdesClientConnectorApi.java} (70%) diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnector.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java similarity index 70% rename from ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnector.java rename to ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java index c6819a132..8cae8ec1a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnector.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java @@ -1,11 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TransferService; -import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; @@ -18,18 +14,16 @@ import static org.springframework.web.reactive.function.server.RequestPredicates.POST; import static org.springframework.web.reactive.function.server.RouterFunctions.route; -public class LdioLdesClientConnector extends LdioLdesClient { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector"; - private static final Logger log = LoggerFactory.getLogger(LdioLdesClientConnector.class); +public class LdioLdesClientConnectorApi { + private static final Logger log = LoggerFactory.getLogger(LdioLdesClientConnectorApi.class); private final TransferService transferService; private final TokenService tokenService; + private final String pipelineName; - public LdioLdesClientConnector(String pipelineName, TransferService transferService, TokenService tokenService, - RequestExecutor edcRequestExecutor, ComponentProperties properties, - ComponentExecutor executor, StatePersistence statePersistence) { - super(NAME, pipelineName, executor, edcRequestExecutor, properties, statePersistence); + public LdioLdesClientConnectorApi(TransferService transferService, TokenService tokenService, String pipelineName) { this.transferService = transferService; this.tokenService = tokenService; + this.pipelineName = pipelineName; } public RouterFunction apiEndpoints() { diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 66a824f23..30c3ddea9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -8,7 +8,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; @@ -21,7 +22,9 @@ @Configuration public class LdioLdesClientConnectorAutoConfig { - @Bean(LdioLdesClientConnector.NAME) + public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector"; + + @Bean(NAME) public LdioInputConfigurator ldioConfigurator() { return new LdioClientConnectorConfigurator(); } @@ -48,11 +51,11 @@ public Object configure(LdiAdapter adapter, ComponentExecutor executor, Componen urlProxy); final StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(properties); - var ldesClientConnector = new LdioLdesClientConnector(pipelineName, transferService, tokenService, edcRequestExecutor, properties, - executor, statePersistence); + LdioLdesClient ldesClient = + new LdioLdesClient(pipelineName, executor, edcRequestExecutor, properties, statePersistence); + ldesClient.start(); - ldesClientConnector.run(); - return ldesClientConnector.apiEndpoints(); + return new LdioLdesClientConnectorApi(transferService, tokenService, pipelineName).apiEndpoints(); } private static EdcUrlProxy getEdcUrlProxy(ComponentProperties properties) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java index 14555e1ba..481e05071 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java @@ -26,8 +26,8 @@ void setup() { transferService = mock(TransferService.class); tokenService = mock(TokenService.class); - final RouterFunction routerFunction = new LdioLdesClientConnector(endpoint, transferService, - tokenService, null, null, null, null).apiEndpoints(); + final RouterFunction routerFunction = + new LdioLdesClientConnectorApi(transferService, tokenService, endpoint).apiEndpoints(); client = WebTestClient .bindToRouterFunction(routerFunction) diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 068853179..72ecf8f97 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -14,6 +14,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.ExecutorService; + +import static java.util.concurrent.Executors.newSingleThreadExecutor; + public class LdioLdesClient extends LdioInput { public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient"; private final Logger log = LoggerFactory.getLogger(LdioLdesClient.class); @@ -23,25 +27,23 @@ public class LdioLdesClient extends LdioInput { private boolean threadRunning = true; - public LdioLdesClient(String pipelineName, ComponentExecutor componentExecutor, - RequestExecutor requestExecutor, ComponentProperties properties, - StatePersistence statePersistence) { - super(NAME, pipelineName, componentExecutor, null); + public LdioLdesClient(String pipelineName, + ComponentExecutor executor, + RequestExecutor requestExecutor, + ComponentProperties properties, + StatePersistence statePersistence) { + super(NAME , pipelineName, executor, null); this.requestExecutor = requestExecutor; this.properties = properties; this.statePersistence = statePersistence; } - public LdioLdesClient(String componentName, String pipelineName, ComponentExecutor executor, - RequestExecutor requestExecutor, ComponentProperties properties, - StatePersistence statePersistence) { - super(componentName, pipelineName, executor, null); - this.requestExecutor = requestExecutor; - this.properties = properties; - this.statePersistence = statePersistence; + public void start() { + final ExecutorService executorService = newSingleThreadExecutor(); + executorService.submit(this::run); } - public void run() { + private void run() { try { log.info("Starting LdesClientRunner run setup"); MemberSupplier memberSupplier = getMemberSupplier(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index cf6cdb398..a1d12a75e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -34,9 +34,9 @@ public LdiComponent configure(LdiAdapter adapter, ComponentExecutor componentExe RequestExecutor requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(properties); - var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, requestExecutor, properties, - statePersistence); - ldesClient.run(); + final var ldesClient = + new LdioLdesClient(pipelineName, componentExecutor, requestExecutor, properties, statePersistence); + ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature index 479388d17..69480e0a0 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature @@ -1,4 +1,4 @@ -Feature: ArchiveFileInIntegrationTest +Feature: LdesClientIntegrationTest As a user I want to be able to use an LDES Client as input for LDIO From 78eac48aabe1f9dbb1571a86584a83d70a09580c Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:04:26 +0100 Subject: [PATCH 4/5] feat: support custom headers (#435) --- docs/_ldio/ldio-core/ldio-http-requester.md | 33 +++++++----- .../StartingTreeNodeFinderSteps.java | 8 +-- .../treenodefetcher/TreeNodeFetcherSteps.java | 7 +-- .../performance/TreeNodeProcessorFactory.java | 6 ++- .../treenodesupplier/MemberSupplierSteps.java | 11 ++-- .../ClientCredentialsConfig.java | 9 +++- .../executor/noauth/ApiKeyConfig.java | 29 ----------- .../executor/noauth/DefaultConfig.java | 9 ++-- .../services/RequestExecutorFactory.java | 50 +++++++++++-------- .../requestexecutor/RequestExecutorSteps.java | 5 -- .../features/request-executor.feature | 2 - .../ldes/ldi/processors/LdesClient.java | 9 +++- .../LdioLdesClientConnectorAutoConfig.java | 3 +- .../LdioRequestExecutorSupplier.java | 38 +++++++++++--- .../RequestExecutorProperties.java | 3 ++ .../LdioRequestExecutorSupplierTest.java | 47 +++++++++++++++-- 16 files changed, 167 insertions(+), 102 deletions(-) delete mode 100644 ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/ApiKeyConfig.java diff --git a/docs/_ldio/ldio-core/ldio-http-requester.md b/docs/_ldio/ldio-core/ldio-http-requester.md index 97acb7ebf..499f3cc92 100644 --- a/docs/_ldio/ldio-core/ldio-http-requester.md +++ b/docs/_ldio/ldio-core/ldio-http-requester.md @@ -10,22 +10,29 @@ Different LDIO components use the Http Requester to make HTTP requests. This requester supports the below config: -| Property | Description | Required | Default | Example | Supported values | -|:------------------------------------|:----------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------| -| auth.type | The type of authentication required by the LDES server | No | NO_AUTH | OAUTH2_CLIENT_CREDENTIALS | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | -| auth.api-key | The api key when using auth.type 'API_KEY' | No | N/A | myKey | String | -| auth.api-key-header | The header for the api key when using auth.type 'API_KEY' | No | X-API-KEY | X-API-KEY | String | -| auth.client-id | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | myId | String | -| auth.client-secret | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | mySecret | String | -| auth.token-endpoint | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | http://localhost:8000/token | HTTP and HTTPS urls | -| retries.enabled | Indicates if the http client should retry http requests when the server cannot be reached. | No | true | true | true or false | -| retries.max | Max number of retries the http client should do when retries.enabled = true | No | 5 | 100 | Integer | -| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No | N/A | 410,451 | Comma seperated list of Integers | -| rate-limit.enabled | Indicates if the http client should limit http requests when calling the server. | No | false | false | true or false | -| rate-limit.max-requests-per-minute | Max number of requests per minute the http client should do when rate-limit.enabled = true | No | 500 | 500 | Integer | +| Property | Description | Required | Default | Example | Supported values | +|:-----------------------------------|:-------------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------| +| auth.type | The type of authentication required by the LDES server | No | NO_AUTH | OAUTH2_CLIENT_CREDENTIALS | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | +| auth.api-key | The api key when using auth.type 'API_KEY' | No | N/A | myKey | String | +| auth.api-key-header | The header for the api key when using auth.type 'API_KEY' | No | X-API-KEY | X-API-KEY | String | +| auth.client-id | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | myId | String | +| auth.client-secret | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | mySecret | String | +| auth.token-endpoint | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | http://localhost:8000/token | HTTP and HTTPS urls | +| retries.enabled | Indicates if the http client should retry http requests when the server cannot be reached. | No | true | true | true or false | +| retries.max | Max number of retries the http client should do when retries.enabled = true | No | 5 | 100 | Integer | +| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No | N/A | 410,451 | Comma seperated list of Integers | +| rate-limit.enabled | Indicates if the http client should limit http requests when calling the server. | No | false | false | true or false | +| rate-limit.max-requests-per-minute | Max number of requests per minute the http client should do when rate-limit.enabled = true | No | 500 | 500 | Integer | +| http.headers.[].key/value | A list of custom http headers can be added. A key and value has to be provided for every header. | No | N/A | role | String | ```yaml config: + http: + headers: + - key: role + value: developer + - key: alt-role + value: programmer auth: type: API_KEY api-key: my-secret diff --git a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java index 8da76dee2..4847e48f6 100644 --- a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java +++ b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java @@ -1,6 +1,6 @@ package ldes.client.startingtreenode; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -10,16 +10,18 @@ import ldes.client.startingtreenode.exception.StartingNodeNotFoundException; import org.apache.jena.riot.RDFLanguages; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class StartingTreeNodeFinderSteps { + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); private StartingTreeNodeFinder startingTreeNodeFinder; private StartingNodeRequest startingNodeRequest; @Given("I have a StartingTreeNodeFinder$") public void initializeCalculator() { - startingTreeNodeFinder = new StartingTreeNodeFinder(new DefaultConfig().createRequestExecutor()); + startingTreeNodeFinder = new StartingTreeNodeFinder(requestExecutorFactory.createNoAuthExecutor()); } @Then("the starting Tree Node of the LDES Stream is the url of the View: {string}") diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java index 3267f66d2..cf61be9a4 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java @@ -1,6 +1,6 @@ package ldes.client.treenodefetcher; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -14,13 +14,14 @@ public class TreeNodeFetcherSteps { + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); private TreeNodeFetcher treeNodeFetcher; private TreeNodeRequest treeNodeRequest; private TreeNodeResponse treeNodeResponse; @Given("I have a TreeNodeFetcher") public void initializeCalculator() { - treeNodeFetcher = new TreeNodeFetcher(new DefaultConfig().createRequestExecutor()); + treeNodeFetcher = new TreeNodeFetcher(requestExecutorFactory.createNoAuthExecutor()); } @When("I create a TreeNodeRequest with Lang {string} and url {string}") @@ -41,7 +42,7 @@ public void theObtainedTreeNodeHasMembersAndRelations(int numberOfMembers, int n @When("I create a TreeNodeRequest with Lang {string} and url {string} and etag {string}") public void iCreateATreeNodeRequestWithLangAndUrlAndEtag(String lang, String url, String etag) { - treeNodeRequest = new TreeNodeRequest(url, RDFLanguages.nameToLang(lang), etag.equals("") ? null : etag); + treeNodeRequest = new TreeNodeRequest(url, RDFLanguages.nameToLang(lang), etag.isEmpty() ? null : etag); } @Then("An UnSupportedOperationException is thrown") diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java index 1330c6856..da0e8a767 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java @@ -1,7 +1,7 @@ package ldes.client.performance; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import ldes.client.treenodesupplier.TreeNodeProcessor; import ldes.client.treenodesupplier.domain.services.MemberRepositoryFactory; import ldes.client.treenodesupplier.domain.services.TreeNodeRecordRepositoryFactory; @@ -18,6 +18,8 @@ class TreeNodeProcessorFactory { + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersistenceStrategy, String url) { final LdesMetaData ldesMetaData = new LdesMetaData(url, Lang.TURTLE); final StatePersistence statePersistence = switch (statePersistenceStrategy) { @@ -26,7 +28,7 @@ TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersiste case FILE -> createFileStatePersistence(); case POSTGRES -> createPostgresPersistence(); }; - final RequestExecutor requestExecutor = new DefaultConfig().createRequestExecutor(); + final RequestExecutor requestExecutor = requestExecutorFactory.createNoAuthExecutor(); return new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java index 5a256db26..3b30da36d 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java @@ -1,6 +1,6 @@ package ldes.client.treenodesupplier; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class MemberSupplierSteps { + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); private TreeNodeProcessor treeNodeProcessor; private TreeNodeRecordRepository treeNodeRecordRepository; private MemberRepository memberRepository; @@ -33,7 +34,7 @@ public class MemberSupplierSteps { // Multi MemberSupplier private final MemberSupplier[] memberSuppliers = new MemberSupplier[2]; - private SuppliedMember[] suppliedMembers = new SuppliedMember[2]; + private final SuppliedMember[] suppliedMembers = new SuppliedMember[2]; @When("I request one member from the MemberSupplier") public void iRequestOneMemberFromTheMemberSupplier() { @@ -60,7 +61,7 @@ public void aStartingUrl(String url) { public void iCreateAProcessor() { treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, new StatePersistence(memberRepository, treeNodeRecordRepository), - new DefaultConfig().createRequestExecutor()); + requestExecutorFactory.createNoAuthExecutor()); } @Then("Member {string} is processed") @@ -161,10 +162,10 @@ private StatePersistence defineStatePersistence(String persistenceStrategy) { public void aStatePersistenceStrategyProcessorAndAStatePersistenceStrategyProcessor(String arg0, String arg1) { memberSuppliers[0] = new MemberSupplier(new TreeNodeProcessor(ldesMetaData, defineStatePersistence(arg0), - new DefaultConfig().createRequestExecutor()), false); + requestExecutorFactory.createNoAuthExecutor()), false); memberSuppliers[1] = new MemberSupplier(new TreeNodeProcessor(ldesMetaData, defineStatePersistence(arg0), - new DefaultConfig().createRequestExecutor()), false); + requestExecutorFactory.createNoAuthExecutor()), false); } @When("I request one member from the MemberSuppliers") diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java index 36877319a..341c3242c 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutorSupplier; +import org.apache.http.Header; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; @@ -10,15 +11,19 @@ import com.github.scribejava.core.oauth.OAuth20Service; import com.github.scribejava.httpclient.apache.ApacheHttpClient; +import java.util.Collection; + import static org.apache.commons.lang3.Validate.notNull; public class ClientCredentialsConfig implements RequestExecutorSupplier { + private final Collection
headers; private final String clientId; private final String secret; private final String tokenEndpoint; - public ClientCredentialsConfig(String clientId, String secret, String tokenEndpoint) { + public ClientCredentialsConfig(Collection
headers, String clientId, String secret, String tokenEndpoint) { + this.headers = headers; this.clientId = notNull(clientId); this.secret = notNull(secret); this.tokenEndpoint = notNull(tokenEndpoint); @@ -31,7 +36,7 @@ public RequestExecutor createRequestExecutor() { private OAuth20Service createService() { final RequestConfig clientConfig = RequestConfig.custom().setRedirectsEnabled(false).build(); final ApacheHttpClient apacheHttpClient = new ApacheHttpClient( - HttpAsyncClientBuilder.create().setDefaultRequestConfig(clientConfig).build()); + HttpAsyncClientBuilder.create().setDefaultRequestConfig(clientConfig).setDefaultHeaders(headers).build()); final DefaultApi20 authorizationApi = createAuthorizationApi(tokenEndpoint); return new ServiceBuilder(clientId) .apiSecret(secret) diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/ApiKeyConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/ApiKeyConfig.java deleted file mode 100644 index 7635cf577..000000000 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/ApiKeyConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutorSupplier; -import org.apache.http.message.BasicHeader; - -import java.util.List; - -import static org.apache.commons.lang3.Validate.notNull; - -public class ApiKeyConfig implements RequestExecutorSupplier { - - private final String apiKeyHeader; - private final String apiKey; - - public ApiKeyConfig(String apiKeyHeader, String apiKey) { - this.apiKeyHeader = notNull(apiKeyHeader, "apiKeyHeader cannot be null"); - this.apiKey = notNull(apiKey, "apiKey cannot be null"); - } - - public RequestExecutor createRequestExecutor() { - return new DefaultConfig().createRequestExecutor(List.of(createBasicHeader())); - } - - private BasicHeader createBasicHeader() { - return new BasicHeader(apiKeyHeader, apiKey); - } - -} diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java index 792c50acc..d3bd09a6d 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java @@ -5,16 +5,17 @@ import org.apache.http.Header; import org.apache.http.impl.client.HttpClientBuilder; -import java.util.ArrayList; import java.util.Collection; public class DefaultConfig implements RequestExecutorSupplier { - public RequestExecutor createRequestExecutor() { - return createRequestExecutor(new ArrayList<>()); + private final Collection
headers; + + public DefaultConfig(Collection
headers) { + this.headers = headers; } - public RequestExecutor createRequestExecutor(Collection
headers) { + public RequestExecutor createRequestExecutor() { return new DefaultRequestExecutor( HttpClientBuilder.create().setDefaultHeaders(headers).disableRedirectHandling().build()); } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java index 967b76e96..5f58da0e9 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java @@ -5,29 +5,39 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.EdcRequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.valueobjects.EdcUrlProxy; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.ApiKeyConfig; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultConfig; +import org.apache.http.Header; + +import java.util.ArrayList; +import java.util.Collection; public class RequestExecutorFactory { - public RequestExecutor createApiKeyExecutor(String keyHeader, String key) { - return new ApiKeyConfig(keyHeader, key).createRequestExecutor(); - } - - public RequestExecutor createNoAuthExecutor() { - return new DefaultConfig().createRequestExecutor(); - } - - public RequestExecutor createClientCredentialsExecutor(String clientId, - String secret, - String tokenEndpoint) { - return new ClientCredentialsConfig(clientId, secret, tokenEndpoint).createRequestExecutor(); - } - - public RequestExecutor createEdcExecutor(RequestExecutor requestExecutor, - TokenService tokenService, - EdcUrlProxy edcUrlProxy) { - return new EdcRequestExecutor(requestExecutor, tokenService, edcUrlProxy); - } + public RequestExecutor createNoAuthExecutor(Collection
headers) { + return new DefaultConfig(headers).createRequestExecutor(); + } + + public RequestExecutor createNoAuthExecutor() { + return createNoAuthExecutor(new ArrayList<>()); + } + + public RequestExecutor createClientCredentialsExecutor(Collection
headers, + String clientId, + String secret, + String tokenEndpoint) { + return new ClientCredentialsConfig(headers, clientId, secret, tokenEndpoint).createRequestExecutor(); + } + + public RequestExecutor createClientCredentialsExecutor(String clientId, + String secret, + String tokenEndpoint) { + return createClientCredentialsExecutor(new ArrayList<>(), clientId, secret, tokenEndpoint); + } + + public RequestExecutor createEdcExecutor(RequestExecutor requestExecutor, + TokenService tokenService, + EdcUrlProxy edcUrlProxy) { + return new EdcRequestExecutor(requestExecutor, tokenService, edcUrlProxy); + } } diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java index 647441116..7adcb97e0 100644 --- a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java @@ -34,11 +34,6 @@ public class RequestExecutorSteps { private Request request; private RequestHeaders requestHeaders = new RequestHeaders(List.of()); - @Given("I have a ApiKeyRequestExecutor") - public void aApiKeyRequestExecutorIsAvailable() { - requestExecutor = factory.createApiKeyExecutor("X-API-KEY", "test123"); - } - @Given("I have a ClientCredentialsRequestExecutor") public void aClientCredentialsRequestExecutorIsAvailable() { requestExecutor = factory.createClientCredentialsExecutor("clientId", "clientSecret", diff --git a/ldi-core/request-executor/src/test/resources/features/request-executor.feature b/ldi-core/request-executor/src/test/resources/features/request-executor.feature index aa17b10c6..bb77090d8 100644 --- a/ldi-core/request-executor/src/test/resources/features/request-executor.feature +++ b/ldi-core/request-executor/src/test/resources/features/request-executor.feature @@ -14,7 +14,6 @@ Feature: RequestExecutor | requestExecutor | endpoint | | DefaultRequestExecutor | http://localhost:10101/200-response-accept-nquads | | ClientCredentialsRequestExecutor | http://localhost:10101/200-response-with-token | - | ApiKeyRequestExecutor | http://localhost:10101/200-response-with-api-key | Scenario Outline: Obtaining the Response of a Post Request Given I have a @@ -29,7 +28,6 @@ Feature: RequestExecutor | requestExecutor | endpoint | | DefaultRequestExecutor | http://localhost:10101/200-post-response-accept-nquads | | ClientCredentialsRequestExecutor | http://localhost:10101/200-post-response-with-token | - | ApiKeyRequestExecutor | http://localhost:10101/200-post-response-with-api-key | Scenario Outline: Obtaining the Response of a redirected Request Given I have a diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClient.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClient.java index 48f1d7a39..d944c2535 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClient.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClient.java @@ -15,6 +15,8 @@ import ldes.client.treenodesupplier.domain.valueobject.LdesMetaData; import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFWriter; @@ -100,7 +102,12 @@ private Retry getRetry(final ProcessContext context) { private RequestExecutor getRequestExecutor(final ProcessContext context) { return switch (getAuthorizationStrategy(context)) { case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(); - case API_KEY -> requestExecutorFactory.createApiKeyExecutor(getApiKeyHeader(context), getApiKey(context)); + case API_KEY -> { + List
headers = List.of( + new BasicHeader(getApiKeyHeader(context), getApiKey(context)) + ); + yield requestExecutorFactory.createNoAuthExecutor(headers); + } case OAUTH2_CLIENT_CREDENTIALS -> requestExecutorFactory.createClientCredentialsExecutor(getOauthClientId(context), getOauthClientSecret(context), getOauthTokenEndpoint(context)); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 30c3ddea9..78339da5b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTransferService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.valueobjects.EdcUrlProxy; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultConfig; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; @@ -36,8 +35,8 @@ public static class LdioClientConnectorConfigurator implements LdioInputConfigur public static final String PROXY_URL_REPLACEMENT = "proxy-url-replacement"; private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); - private final RequestExecutor baseRequestExecutor = new DefaultConfig().createRequestExecutor(); private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutor baseRequestExecutor = requestExecutorFactory.createNoAuthExecutor(); @Override public Object configure(LdiAdapter adapter, ComponentExecutor executor, ComponentProperties properties) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java index 66b29c7de..e33611c2b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java @@ -9,6 +9,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.github.resilience4j.ratelimiter.RateLimiter; import io.github.resilience4j.retry.Retry; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; import java.util.ArrayList; import java.util.List; @@ -17,6 +19,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy.NO_AUTH; import static be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.RequestExecutorProperties.*; +import static org.apache.commons.lang3.ObjectUtils.isNotEmpty; /** * Creates a RequestExecutor based on the config provided using LDIO @@ -69,17 +72,24 @@ private Retry getRetry(ComponentProperties props) { private RequestExecutor getBaseRequestExecutor(ComponentProperties componentProperties) { Optional authentication = AuthStrategy .from(componentProperties.getOptionalProperty(AUTH_TYPE).orElse(NO_AUTH.name())); + final List
headers = getHttpHeaders(componentProperties); + if (authentication.isPresent()) { return switch (authentication.get()) { - case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(); - case API_KEY -> - requestExecutorFactory - .createApiKeyExecutor( - componentProperties.getOptionalProperty(API_KEY_HEADER) - .orElse(DEFAULT_API_KEY_HEADER), - componentProperties.getProperty(API_KEY)); + case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(headers); + case API_KEY -> { + String apiKeyHeader = componentProperties + .getOptionalProperty(API_KEY_HEADER) + .orElse(DEFAULT_API_KEY_HEADER); + String apiKey = componentProperties.getProperty(API_KEY); + + List
headersWithApiKey = new ArrayList<>(headers); + headersWithApiKey.add(new BasicHeader(apiKeyHeader, apiKey)); + yield requestExecutorFactory.createNoAuthExecutor(headersWithApiKey); + } case OAUTH2_CLIENT_CREDENTIALS -> requestExecutorFactory.createClientCredentialsExecutor( + headers, componentProperties.getProperty(CLIENT_ID), componentProperties.getProperty(CLIENT_SECRET), componentProperties.getProperty(TOKEN_ENDPOINT)); @@ -89,4 +99,18 @@ private RequestExecutor getBaseRequestExecutor(ComponentProperties componentProp + componentProperties.getOptionalProperty(AUTH_TYPE).orElse("No auth type provided")); } + private List
getHttpHeaders(ComponentProperties componentProperties) { + final ComponentProperties headers = componentProperties.extractNestedProperties(HTTP_HEADERS); + final List
result = new ArrayList<>(); + for (int i = 0; isNotEmpty(headers.extractNestedProperties(String.valueOf(i)).getConfig()); i++) { + ComponentProperties headerProperties = headers.extractNestedProperties(String.valueOf(i)); + BasicHeader basicHeader = new BasicHeader( + headerProperties.getProperty(HTTP_HEADERS_KEY), + headerProperties.getProperty(HTTP_HEADERS_VALUE) + ); + result.add(basicHeader); + } + return result; + } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java index 8a4839b9e..552710e75 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java @@ -5,6 +5,9 @@ public class RequestExecutorProperties { private RequestExecutorProperties() { } + public static final String HTTP_HEADERS = "http.headers"; + public static final String HTTP_HEADERS_KEY = "key"; + public static final String HTTP_HEADERS_VALUE = "value"; public static final String HTTP_METHOD = "http.method"; public static final String HTTP_CONTENT_TYPE = "http.content-type"; diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java index 8abf9e52b..82071a6d8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java @@ -7,6 +7,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.github.resilience4j.ratelimiter.RateLimiter; import io.github.resilience4j.retry.Retry; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -15,10 +17,13 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import java.util.List; import java.util.Map; +import java.util.Objects; import static be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.RequestExecutorProperties.*; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -49,6 +54,30 @@ void shouldReturnRetryExecutorWithDefaults_whenNoProperties() { } } + /** + * Required for test purposes. The mock compares on equals which is not implemented in BasicHeader. + */ + static class BasicHeaderWithEquals extends BasicHeader { + + public BasicHeaderWithEquals(String name, String value) { + super(name, value); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Header header)) return false; + return Objects.equals(getName(), header.getName()) && Objects.equals(getValue(), header.getValue()); + } + + @Override + public int hashCode() { + return Objects.hash(getName(), getValue()); + } + + } + + @Test void shouldReturnRetryExecutorWithConfiguredProperties_whenPropertiesConfigured() { String maxRetries = "10"; @@ -57,9 +86,19 @@ void shouldReturnRetryExecutorWithConfiguredProperties_whenPropertiesConfigured( STATUSES_TO_RETRY, "400,404", AUTH_TYPE, AuthStrategy.API_KEY.name(), API_KEY_HEADER, "key-header", - API_KEY, "key")); + API_KEY, "key", + HTTP_HEADERS + ".0.key", "header-key", + HTTP_HEADERS + ".0.value", "header-value", + HTTP_HEADERS + ".1.key", "other-header-key", + HTTP_HEADERS + ".1.value", "other-header-value")); RequestExecutor requestExecutor = mock(RequestExecutor.class); - when(requestExecutorFactory.createApiKeyExecutor("key-header", "key")).thenReturn(requestExecutor); + List
expectedHeaders = + List.of( + new BasicHeaderWithEquals("header-key", "header-value"), + new BasicHeaderWithEquals("other-header-key", "other-header-value"), + new BasicHeaderWithEquals("key-header", "key") + ); + when(requestExecutorFactory.createNoAuthExecutor(expectedHeaders)).thenReturn(requestExecutor); RequestExecutorDecorator requestExecutorDecorator = mock(RequestExecutorDecorator.class); when(requestExecutorDecorator.with((Retry) any())).thenReturn(requestExecutorDecorator); when(requestExecutorDecorator.with((RateLimiter) any())).thenReturn(requestExecutorDecorator); @@ -104,7 +143,7 @@ void shouldReturnNonRetryExecutorWithConfiguredProperties_whenPropertiesConfigur CLIENT_SECRET, "secret", TOKEN_ENDPOINT, "token")); RequestExecutor requestExecutor = mock(RequestExecutor.class); - when(requestExecutorFactory.createClientCredentialsExecutor("client", "secret", "token")) + when(requestExecutorFactory.createClientCredentialsExecutor(List.of(), "client", "secret", "token")) .thenReturn(requestExecutor); RequestExecutor result = requestExecutorSupplier.getRequestExecutor(properties); From ba165ef73e0c598942742223d2d3d522882d52ce Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:55:03 +0100 Subject: [PATCH 5/5] fix: broken tracing (#430) --- .../ClientCredentialsConfig.java | 4 ++-- .../ldes/ldio/modules/DummyIn.java | 2 +- ldi-orchestrator/ldio-common/pom.xml | 4 ++++ .../ldio/config/ObserveConfiguration.java | 15 +++++------- .../ldes/ldio/types/LdioInput.java | 10 ++++---- .../ldes/ldio/types/LdioTransformer.java | 24 ++++++++----------- .../ldes/ldio/LdioArchiveFileIn.java | 5 ++-- .../config/LdioArchiveFileInAutoConfig.java | 6 +++-- .../ldes/ldio/ArchiveFileInITSteps.java | 3 ++- .../ldes/ldio/HttpInputPoller.java | 9 +++---- .../config/HttpInputPollerAutoConfig.java | 13 +++++++--- .../ldio/HttpInputPollerAutoConfigTest.java | 4 ++-- .../ldes/ldio/HttpInputPollerTest.java | 12 +++++----- .../ldes/ldio/LdioHttpIn.java | 5 ++-- .../ldio/config/LdioHttpInAutoConfig.java | 20 +++++++++------- .../ldes/ldio/LdioHttpInputTest.java | 2 +- .../ldes/ldio/LdioKafkaIn.java | 5 ++-- .../ldio/config/LdioKafkaInAutoConfig.java | 15 ++++++++---- .../config/KafkaInIntegrationTestSteps.java | 2 +- .../config/LdioKafkaInAutoConfigTest.java | 7 +++--- .../ldes/ldio/LdioLdesClientConnectorApi.java | 19 ++++++++------- .../LdioLdesClientConnectorAutoConfig.java | 12 +++++++--- .../ldes/ldio/LdioLdesClient.java | 17 +++++++------ .../ldio/config/LdioLdesClientAutoConfig.java | 14 +++++++---- .../ldio/config/LdioLdesClientITSteps.java | 2 +- 25 files changed, 134 insertions(+), 97 deletions(-) diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java index 341c3242c..5aec3442e 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java @@ -6,13 +6,13 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import java.util.Collection; + import com.github.scribejava.core.builder.ServiceBuilder; import com.github.scribejava.core.builder.api.DefaultApi20; import com.github.scribejava.core.oauth.OAuth20Service; import com.github.scribejava.httpclient.apache.ApacheHttpClient; -import java.util.Collection; - import static org.apache.commons.lang3.Validate.notNull; public class ClientCredentialsConfig implements RequestExecutorSupplier { diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java index be3b992ea..0f7a77a3f 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java @@ -8,7 +8,7 @@ public class DummyIn extends LdioInput { private int counter = 0; public DummyIn(ComponentExecutor executor, LdiAdapter adapter) { - super("DummyIn", "test", executor, adapter); + super("DummyIn", "test", executor, adapter, null); } public void sendData() { diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index 293e92929..f8ba58912 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -31,6 +31,10 @@ 1.11.5 compile + + org.springframework.boot + spring-boot-starter-aop + \ No newline at end of file diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ObserveConfiguration.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ObserveConfiguration.java index 6a6cafbae..2bd18f8e7 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ObserveConfiguration.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ObserveConfiguration.java @@ -1,18 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; import io.micrometer.observation.ObservationRegistry; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +@Configuration public class ObserveConfiguration { public static final String ERROR_TEMPLATE = "ERROR - when='{}', problem='{}'"; - private static ObservationRegistry observationRegistry; - private ObserveConfiguration() {} - - public static ObservationRegistry observationRegistry() { - if (observationRegistry == null) { - observationRegistry = ObservationRegistry.create(); - } - - return observationRegistry; + @Bean + public ObservationRegistry observationRegistry() { + return ObservationRegistry.create(); } } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java index 55c53aae6..f9661c4ea 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java @@ -40,12 +40,12 @@ public abstract class LdioInput implements LdiComponent { * @param adapter Instance of the LDI Adapter. Facilitates transforming the input * data to a linked data model (RDF). */ - protected LdioInput(String componentName, String pipelineName, ComponentExecutor executor, LdiAdapter adapter) { + protected LdioInput(String componentName, String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry) { this.componentName = componentName; this.pipelineName = pipelineName; this.executor = executor; this.adapter = adapter; - this.observationRegistry = ObserveConfiguration.observationRegistry(); + this.observationRegistry = observationRegistry; Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(0); } @@ -55,12 +55,11 @@ protected void processInput(String content, String contentType) { protected void processInput(LdiAdapter.Content content) { Observation.createNotStarted(this.componentName, observationRegistry) - .contextualName(this.pipelineName) .observe(() -> { try { adapter.apply(content).forEach(this::processModel); } catch (Exception e) { - log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, this.pipelineName, e.getMessage()); + log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, this.pipelineName + ":processInput", e.getMessage()); throw e; } }); @@ -69,12 +68,11 @@ protected void processInput(LdiAdapter.Content content) { protected void processModel(Model model) { Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(); Observation.createNotStarted(this.componentName, observationRegistry) - .contextualName(this.pipelineName) .observe(() -> { try { executor.transformLinkedData(model); } catch (Exception e) { - log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, this.pipelineName, e.getMessage()); + log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, this.pipelineName + ":processModel", e.getMessage()); throw e; } }); diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java index cd2c8a7f3..c27127636 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.types; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.ObserveConfiguration; -import io.micrometer.observation.Observation; +import io.micrometer.observation.annotation.Observed; import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,25 +9,21 @@ import java.util.List; public abstract class LdioTransformer { - private LdioTransformer nextProcessor; private final Logger log = LoggerFactory.getLogger(this.getClass()); public abstract void apply(Model model); + @Observed protected void next(Model model) { - Observation.createNotStarted(this.getClass().getSimpleName(), ObserveConfiguration.observationRegistry()) - .contextualName("transform") - .observe(() -> { - try { - if (nextProcessor != null) { - nextProcessor.apply(model); - } - } catch (Exception e) { - log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, "transformModel", e.getMessage()); - throw e; - } - }); + try { + if (nextProcessor != null) { + nextProcessor.apply(model); + } + } catch (Exception e) { + log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, "transformModel", e.getMessage()); + throw e; + } } public static LdioTransformer link(LdioTransformer first, List chain) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java index 19c78fd27..88452a7b5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import io.micrometer.observation.ObservationRegistry; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParser; @@ -14,8 +15,8 @@ public class LdioArchiveFileIn extends LdioInput { private final ArchiveFileCrawler archiveFileCrawler; private final Lang sourceFormat; - public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, ArchiveFileCrawler crawler, Lang source) { - super(NAME, pipelineName, executor, null); + public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, ObservationRegistry observationRegistry, ArchiveFileCrawler crawler, Lang source) { + super(NAME, pipelineName, executor, null, observationRegistry); this.archiveFileCrawler = crawler; this.sourceFormat = source; log.info("Starting with crawling the archive."); diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java index 9677c2bf0..045a26a38 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioArchiveFileInAutoConfig.java @@ -7,6 +7,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioArchiveFileIn; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.micrometer.observation.ObservationRegistry; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; import org.springframework.context.annotation.Bean; @@ -22,8 +23,9 @@ public class LdioArchiveFileInAutoConfig { public static final String ARCHIVE_ROOT_DIR_PROP = "archive-root-dir"; public static final String SOURCE_FORMAT = "source-format"; + @SuppressWarnings("java:S6830") @Bean(LdioArchiveFileIn.NAME) - public LdioInputConfigurator ldiArchiveFileInConfigurator() { + public LdioInputConfigurator ldiArchiveFileInConfigurator(ObservationRegistry observationRegistry) { return new LdioInputConfigurator() { @Override public LdiComponent configure(LdiAdapter adapter, @@ -33,7 +35,7 @@ public LdiComponent configure(LdiAdapter adapter, Lang hintLang = getSourceFormat(config); String pipelineName = config.getProperty(PIPELINE_NAME); - return new LdioArchiveFileIn(pipelineName, executor, archiveFileCrawler, hintLang); + return new LdioArchiveFileIn(pipelineName, executor, observationRegistry, archiveFileCrawler, hintLang); } private Path getArchiveDirectoryPath(ComponentProperties config) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java index 9b98858a6..18312d49c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFileInITSteps.java @@ -5,6 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; +import io.micrometer.observation.ObservationRegistry; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.RDFParser; @@ -28,7 +29,7 @@ public void iCreateAnArchiveFileInComponentWithArchiveDir(String archiveDir) { ComponentExecutor componentExecutor = linkedDataModel -> members.add(linkedDataModel); var props = new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", ARCHIVE_ROOT_DIR_PROP, separatorsToSystem(archiveDir))); - var ldioInputConfigurator = new LdioArchiveFileInAutoConfig().ldiArchiveFileInConfigurator(); + var ldioInputConfigurator = new LdioArchiveFileInAutoConfig().ldiArchiveFileInConfigurator(ObservationRegistry.create()); ldioInputConfigurator.configure(null, componentExecutor, props); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java index 233a87d50..c2ea0562f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java @@ -10,6 +10,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.MissingHeaderException; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.UnsuccesfulPollingException; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import io.micrometer.observation.ObservationRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatusCode; @@ -28,9 +29,9 @@ public class HttpInputPoller extends LdioInput { private static final Logger log = LoggerFactory.getLogger(HttpInputPoller.class); private static final String CONTENT_TYPE = "Content-Type"; - public HttpInputPoller(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, List endpoints, - boolean continueOnFail, RequestExecutor requestExecutor) { - super(NAME, pipelineName, executor, adapter); + public HttpInputPoller(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, List endpoints, + boolean continueOnFail, RequestExecutor requestExecutor) { + super(NAME, pipelineName, executor, adapter, observationRegistry); this.requestExecutor = requestExecutor; this.requests = endpoints.stream().map(endpoint -> new GetRequest(endpoint, RequestHeaders.empty())).toList(); this.continueOnFail = continueOnFail; @@ -59,7 +60,7 @@ private void executeRequest(Request request) { Response response = requestExecutor.execute(request); - log.debug("{} {} {}", request.getMethod() , request.getUrl() , response.getHttpStatus()); + log.debug("{} {} {}", request.getMethod(), request.getUrl(), response.getHttpStatus()); if (HttpStatusCode.valueOf(response.getHttpStatus()).is2xxSuccessful()) { String contentType = response.getFirstHeaderValue(CONTENT_TYPE) diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java index 7cb14f29b..f73b7a85b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java @@ -6,6 +6,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.micrometer.observation.ObservationRegistry; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -20,12 +21,18 @@ public class HttpInputPollerAutoConfig { private static final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier(); + @SuppressWarnings("java:S6830") @Bean(HttpInputPoller.NAME) - public HttpInputPollerConfigurator httpInputPollerConfigurator() { - return new HttpInputPollerConfigurator(); + public HttpInputPollerConfigurator httpInputPollerConfigurator(ObservationRegistry observationRegistry) { + return new HttpInputPollerConfigurator(observationRegistry); } public static class HttpInputPollerConfigurator implements LdioInputConfigurator { + private final ObservationRegistry observationRegistry; + + public HttpInputPollerConfigurator(ObservationRegistry observationRegistry) { + this.observationRegistry = observationRegistry; + } @Override public HttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, @@ -45,7 +52,7 @@ public HttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, } var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); - var httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, endpoints, continueOnFail, requestExecutor); + var httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, observationRegistry, endpoints, continueOnFail, requestExecutor); httpInputPoller.schedulePoller(seconds); return httpInputPoller; diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java index 8543c38d5..e91549018 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java @@ -32,7 +32,7 @@ class HttpInputPollerAutoConfigTest { void when_ValidConfig() { try (MockedConstruction ignored = mockConstruction(HttpInputPoller.class)) { HttpInputPoller poller = new HttpInputPollerAutoConfig() - .httpInputPollerConfigurator() + .httpInputPollerConfigurator(null) .configure(adapter, executor, createDefaultTestConfig()); verify(poller, times(1)).schedulePoller(1); } @@ -42,7 +42,7 @@ void when_ValidConfig() { @ArgumentsSource(InvalidIntervalArgumentsProvider.class) void whenInvalidIntervalConfigured_thenCatchException(String interval) { Executable configurePoller = () -> new HttpInputPollerAutoConfig() - .httpInputPollerConfigurator() + .httpInputPollerConfigurator(null) .configure(adapter, executor, createConfig(BASE_URL + ENDPOINT, interval, "false")); assertThrows(IllegalArgumentException.class, configurePoller); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java index 2007ccb86..a8c45ef13 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java @@ -43,7 +43,7 @@ void setUp() { .thenReturn(Stream.of()) .thenReturn(Stream.of()); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor); + httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor); } @Test @@ -59,7 +59,7 @@ void testClientPolling() { void whenPolling_andMissesHeader() { stubFor(get(ENDPOINT).willReturn(ok().withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); + httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); Executable polling = () -> httpInputPoller.poll(); assertThrows(MissingHeaderException.class, polling); @@ -82,7 +82,7 @@ void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldSti stubFor(get(ENDPOINT).willReturn(serverError().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); String otherEndpoint = "/other-resource"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), + httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), true, noAuthExecutor); httpInputPoller.poll(); @@ -97,7 +97,7 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse() { stubFor(get(endpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); String otherEndpoint = "/other-endpoint"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), + httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), true, noAuthExecutor); httpInputPoller.schedulePoller(1); @@ -123,7 +123,7 @@ void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling() { void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling() { stubFor(get(ENDPOINT).willReturn(forbidden())); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); + httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); httpInputPoller.schedulePoller(1); Mockito.verify(adapter, after(2000).never()).apply(any()); @@ -133,7 +133,7 @@ void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling() { @Test void when_EndpointDoesNotExist_Then_NoDataIsSent() { String wrongEndpoint = "/non-existing-resource"; - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, List.of(BASE_URL + wrongEndpoint), true, + httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + wrongEndpoint), true, noAuthExecutor); httpInputPoller.poll(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpIn.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpIn.java index 2d6b7ac39..ba66f2b31 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpIn.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import io.micrometer.observation.ObservationRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; @@ -20,8 +21,8 @@ public class LdioHttpIn extends LdioInput { public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn"; private static final Logger log = LoggerFactory.getLogger(LdioHttpIn.class); - public LdioHttpIn(String pipelineName, ComponentExecutor executor, LdiAdapter adapter) { - super(NAME, pipelineName, executor, adapter); + public LdioHttpIn(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry) { + super(NAME, pipelineName, executor, adapter, observationRegistry); } public RouterFunction mapping() { diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java index 4a2a60c33..cc8bd9499 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java @@ -5,7 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import org.springframework.beans.factory.annotation.Autowired; +import io.micrometer.observation.ObservationRegistry; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,23 +17,27 @@ public class LdioHttpInAutoConfig { @SuppressWarnings("java:S6830") @Bean(LdioHttpIn.NAME) - public LdioHttpInConfigurator ldioConfigurator() { - return new LdioHttpInConfigurator(); + public LdioHttpInConfigurator ldioConfigurator(ConfigurableApplicationContext configContext, + ObservationRegistry observationRegistry) { + return new LdioHttpInConfigurator(observationRegistry); } public static class LdioHttpInConfigurator implements LdioInputConfigurator { - @Autowired - ConfigurableApplicationContext configContext; + private final ObservationRegistry observationRegistry; + + public LdioHttpInConfigurator(ObservationRegistry observationRegistry) { + this.observationRegistry = observationRegistry; + } @Override public Object configure(LdiAdapter adapter, - ComponentExecutor executor, - ComponentProperties config) { + ComponentExecutor executor, + ComponentProperties config) { String pipelineName = config.getProperty(PIPELINE_NAME); verifyAdapterPresent(pipelineName, adapter); - LdioHttpIn ldioHttpIn = new LdioHttpIn(pipelineName, executor, adapter); + LdioHttpIn ldioHttpIn = new LdioHttpIn(pipelineName, executor, adapter, observationRegistry); return ldioHttpIn.mapping(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java index d290b5bf1..32276dc44 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java @@ -31,7 +31,7 @@ void setup() { when(adapter.apply(any())).thenReturn(Stream.empty()); - RouterFunction routerFunction = (RouterFunction) new LdioHttpInAutoConfig.LdioHttpInConfigurator() + RouterFunction routerFunction = (RouterFunction) new LdioHttpInAutoConfig.LdioHttpInConfigurator(null) .configure(adapter, executor, new ComponentProperties(Map.of(PIPELINE_NAME, endpoint))); client = WebTestClient diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java index beb4ddde4..af3f92341 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import io.micrometer.observation.ObservationRegistry; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; @@ -25,8 +26,8 @@ public class LdioKafkaIn extends LdioInput implements MessageListener(getConsumerConfig(config)); ContainerProperties containerProps = new ContainerProperties(config.getProperty(TOPICS).split(",")); containerProps.setMessageListener(ldioKafkaIn); @@ -68,7 +75,7 @@ private Map getConsumerConfig(ComponentProperties config) { props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); var authStrategy = KafkaAuthStrategy.from(config.getOptionalProperty(SECURITY_PROTOCOL) - .orElse(KafkaAuthStrategy.NO_AUTH.name())) + .orElse(KafkaAuthStrategy.NO_AUTH.name())) .orElseThrow(() -> new SecurityProtocolNotSupportedException(SECURITY_PROTOCOL)); if (KafkaAuthStrategy.SASL_SSL_PLAIN.equals(authStrategy)) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java index a9cb05de8..d04cf2bb4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java @@ -56,7 +56,7 @@ public void iCreateAnLdioKafkaInComponent() { adapterResult.add(content); return Stream.of(toModel(content)); }; - var ldioKafkaInConfigurator = new LdioKafkaInAutoConfig().ldioConfigurator(); + var ldioKafkaInConfigurator = new LdioKafkaInAutoConfig().ldioConfigurator(null); var ldioKafkaInContainer = (KafkaMessageListenerContainer) ldioKafkaInConfigurator .configure(adapter, componentExecutor, properties); ldioKafkaInContainer.start(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java index 7fa4846c3..8cc475b1d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioKafkaInAutoConfigTest.java @@ -4,6 +4,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.config.KafkaInConfigKeys; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioKafkaInAutoConfig; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.micrometer.observation.ObservationRegistry; import org.junit.jupiter.api.Test; import java.util.HashMap; @@ -18,7 +19,7 @@ class LdioKafkaInAutoConfigTest { @Test void shouldThrowExceptionWhenInvalidAuthConfig() { - var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(); + var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(ObservationRegistry.create()); Map config = getBasicConfig(); config.put(KafkaInConfigKeys.SECURITY_PROTOCOL, "Fantasy protocol"); @@ -33,7 +34,7 @@ void shouldThrowExceptionWhenInvalidAuthConfig() { @Test void shouldNotThrowExceptionWhenNoAuthConfig() { - var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(); + var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(ObservationRegistry.create()); Map config = getBasicConfig(); @@ -43,7 +44,7 @@ void shouldNotThrowExceptionWhenNoAuthConfig() { @Test void shouldNotThrowExceptionWhenSaslSslPlain() { - var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(); + var configurator = new LdioKafkaInAutoConfig.LdioKafkaInConfigurator(ObservationRegistry.create()); Map config = getBasicConfig(); config.put(KafkaInConfigKeys.SECURITY_PROTOCOL, KafkaAuthStrategy.SASL_SSL_PLAIN.name()); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java index 8cae8ec1a..bdcf6c5a5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java @@ -20,6 +20,7 @@ public class LdioLdesClientConnectorApi { private final TokenService tokenService; private final String pipelineName; + @SuppressWarnings("java:S107") public LdioLdesClientConnectorApi(TransferService transferService, TokenService tokenService, String pipelineName) { this.transferService = transferService; this.tokenService = tokenService; @@ -34,15 +35,15 @@ public RouterFunction apiEndpoints() { .doOnNext(tokenService::updateToken) .flatMap(body -> ServerResponse.accepted().build()); }).andRoute(POST("/%s/transfer".formatted(pipelineName)), - request -> { - logIncomingRequest(request); - return request.bodyToMono(String.class) - .flatMap(requestString -> { - var response = transferService.startTransfer(requestString).getBody() - .orElse(""); - return ServerResponse.accepted().body(Mono.just(response), String.class); - }); - }); + request -> { + logIncomingRequest(request); + return request.bodyToMono(String.class) + .flatMap(requestString -> { + var response = transferService.startTransfer(requestString).getBody() + .orElse(""); + return ServerResponse.accepted().body(Mono.just(response), String.class); + }); + }); } private void logIncomingRequest(ServerRequest request) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 78339da5b..8526cba26 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -11,6 +11,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,8 +25,8 @@ public class LdioLdesClientConnectorAutoConfig { public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector"; @Bean(NAME) - public LdioInputConfigurator ldioConfigurator() { - return new LdioClientConnectorConfigurator(); + public LdioInputConfigurator ldioConfigurator(ObservationRegistry observationRegistry) { + return new LdioClientConnectorConfigurator(observationRegistry); } public static class LdioClientConnectorConfigurator implements LdioInputConfigurator { @@ -33,11 +34,16 @@ public static class LdioClientConnectorConfigurator implements LdioInputConfigur public static final String CONNECTOR_TRANSFER_URL = "connector-transfer-url"; public static final String PROXY_URL_TO_REPLACE = "proxy-url-to-replace"; public static final String PROXY_URL_REPLACEMENT = "proxy-url-replacement"; + private final ObservationRegistry observationRegistry; private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); private final RequestExecutor baseRequestExecutor = requestExecutorFactory.createNoAuthExecutor(); + public LdioClientConnectorConfigurator(ObservationRegistry observationRegistry) { + this.observationRegistry = observationRegistry; + } + @Override public Object configure(LdiAdapter adapter, ComponentExecutor executor, ComponentProperties properties) { final String pipelineName = properties.getProperty(PIPELINE_NAME); @@ -51,7 +57,7 @@ public Object configure(LdiAdapter adapter, ComponentExecutor executor, Componen final StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(properties); LdioLdesClient ldesClient = - new LdioLdesClient(pipelineName, executor, edcRequestExecutor, properties, statePersistence); + new LdioLdesClient(pipelineName, executor, observationRegistry, edcRequestExecutor, properties, statePersistence); ldesClient.start(); return new LdioLdesClientConnectorApi(transferService, tokenService, pipelineName).apiEndpoints(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 72ecf8f97..4bb92539a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -4,6 +4,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.MemberSupplier; import ldes.client.treenodesupplier.TreeNodeProcessor; import ldes.client.treenodesupplier.domain.valueobject.EndOfLdesException; @@ -28,16 +29,18 @@ public class LdioLdesClient extends LdioInput { private boolean threadRunning = true; public LdioLdesClient(String pipelineName, - ComponentExecutor executor, - RequestExecutor requestExecutor, - ComponentProperties properties, - StatePersistence statePersistence) { - super(NAME , pipelineName, executor, null); + ComponentExecutor componentExecutor, + ObservationRegistry observationRegistry, + RequestExecutor requestExecutor, + ComponentProperties properties, + StatePersistence statePersistence) { + super(NAME, pipelineName, componentExecutor, null, observationRegistry); this.requestExecutor = requestExecutor; this.properties = properties; this.statePersistence = statePersistence; } + @SuppressWarnings("java:S2095") public void start() { final ExecutorService executorService = newSingleThreadExecutor(); executorService.submit(this::run); @@ -72,8 +75,8 @@ private MemberSupplier getMemberSupplier() { } private TreeNodeProcessor getTreeNodeProcessor(StatePersistence statePersistenceStrategy, - RequestExecutor requestExecutor, - LdesMetaData ldesMetaData) { + RequestExecutor requestExecutor, + LdesMetaData ldesMetaData) { return new TreeNodeProcessor(ldesMetaData, statePersistenceStrategy, requestExecutor); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index a1d12a75e..8a1dfd156 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -8,6 +8,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -18,14 +19,19 @@ public class LdioLdesClientAutoConfig { @SuppressWarnings("java:S6830") @Bean(LdioLdesClient.NAME) - public LdioInputConfigurator ldioConfigurator() { - return new LdioLdesClientConfigurator(); + public LdioInputConfigurator ldioConfigurator(ObservationRegistry observationRegistry) { + return new LdioLdesClientConfigurator(observationRegistry); } public static class LdioLdesClientConfigurator implements LdioInputConfigurator { private final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier(); private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); + private final ObservationRegistry observationRegistry; + + public LdioLdesClientConfigurator(ObservationRegistry observationRegistry) { + this.observationRegistry = observationRegistry; + } @Override public LdiComponent configure(LdiAdapter adapter, ComponentExecutor componentExecutor, @@ -34,8 +40,8 @@ public LdiComponent configure(LdiAdapter adapter, ComponentExecutor componentExe RequestExecutor requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(properties); - final var ldesClient = - new LdioLdesClient(pipelineName, componentExecutor, requestExecutor, properties, statePersistence); + var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, observationRegistry, requestExecutor, + properties, statePersistence); ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java index 83f972417..a69df6b6e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java @@ -34,7 +34,7 @@ public void iStartAnLdesLdioInComponentWithUrl(String url) { ComponentExecutor componentExecutor = linkedDataModel -> members.add(linkedDataModel); var props = new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", LdioLdesClientProperties.URL, wireMockServer.baseUrl() + url)); - var ldioInputConfigurator = new LdioLdesClientAutoConfig().ldioConfigurator(); + var ldioInputConfigurator = new LdioLdesClientAutoConfig().ldioConfigurator(null); ldioInputConfigurator.configure(null, componentExecutor, props); }