diff --git a/docs/_core/ldi-inputs/ldes-client.md b/docs/_core/ldi-inputs/ldes-client.md index 7ae652d35..ef43e3522 100644 --- a/docs/_core/ldi-inputs/ldes-client.md +++ b/docs/_core/ldi-inputs/ldes-client.md @@ -16,4 +16,6 @@ It will be ignored when an attempt is made to queue a known immutable fragment. Processed members of mutable fragments are also kept in state. They are ignored if presented more than once. -[VSDS Tech Docs]: https://informatievlaanderen.github.io/VSDS-Tech-Docs/introduction/LDES_client +Within a fragment, members can be ordered based on a timestamp. The path to this timestamp has to be configured. If this path is missing, the members are ordered randomly. + +[VSDS Tech Docs]: https://informatievlaanderen.github.io/VSDS-Tech-Docs/introduction/LDES_client \ No newline at end of file diff --git a/docs/_ldio/examples/ex1-basicHttp.md b/docs/_ldio/examples/ex1-basicHttp.md index 41251ae3d..ef229c0a1 100644 --- a/docs/_ldio/examples/ex1-basicHttp.md +++ b/docs/_ldio/examples/ex1-basicHttp.md @@ -23,11 +23,11 @@ orchestrator: - name: data description: "This pipeline uses a HTTP listener to read incoming RDF data and writes them to the console" input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn + name: Ldio:HttpIn adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter + name: Ldio:RdfAdapter outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioConsoleOut + - name: Ldio:ConsoleOut ```` ## Execution diff --git a/docs/_ldio/examples/ex2-scrape-api.md b/docs/_ldio/examples/ex2-scrape-api.md index 2bdd62c0c..797b1de1b 100644 --- a/docs/_ldio/examples/ex2-scrape-api.md +++ b/docs/_ldio/examples/ex2-scrape-api.md @@ -70,23 +70,23 @@ orchestrator: pipelines: - name: data input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + name: Ldio:HttpInPoller config: url: https://www.cheapshark.com/api/1.0/deals?pageSize=1000 interval: PT30M adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + name: Ldio:RmlAdapter config: mapping: "mapping.ttl" transformers: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator + - name: Ldio:VersionObjectCreator config: date-observed-property: "http://www.cheapshark.com/lastChange" member-type: "http://www.cheapshark.com/GameDeal" generatedAt-property: "https://w3id.org/ldes#timestampPath" versionOf-property: "https://w3id.org/ldes#versionOfPath" outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioConsoleOut + - name: Ldio:ConsoleOut config: content-type: text/turtle ``` diff --git a/docs/_ldio/examples/ex3-enrich-model.md b/docs/_ldio/examples/ex3-enrich-model.md index daca7f977..cd445b59b 100644 --- a/docs/_ldio/examples/ex3-enrich-model.md +++ b/docs/_ldio/examples/ex3-enrich-model.md @@ -68,21 +68,21 @@ orchestrator: pipelines: - name: "to-graph" input: - name: "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn" + name: "Ldio:HttpIn" adapter: - name: "be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter" + name: "Ldio:RdfAdapter" outputs: - - name: "be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser" + - name: "Ldio:RepositoryMaterialiser" config: sparql-host: http://localhost:8081/rdf4j-server repository-id: test - name: "enriched" input: - name: "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn" + name: "Ldio:HttpIn" adapter: - name: "be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter" + name: "Ldio:RdfAdapter" transformers: - - name: "be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer" + - name: "Ldio:SparqlConstructTransformer" config: query: " PREFIX schema: @@ -100,7 +100,7 @@ orchestrator: } " outputs: - - name: "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioConsoleOut" + - name: "Ldio:ConsoleOut" ``` ## Execution diff --git a/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md b/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md index f57968160..68c44252b 100644 --- a/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md +++ b/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md @@ -5,7 +5,8 @@ title: Json To JsonLd Transformer --- # LDIO Json To JsonLd Transformer -***be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter*** + +***Ldio:JsonToLdAdapter*** An LDIO wrapper component for the [LDI Json To JsonLd building block](../../_core/ldi-adapters/json-to-json-ld) diff --git a/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md b/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md index f9755e324..062621964 100644 --- a/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md +++ b/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md @@ -5,7 +5,8 @@ title: NGSIv2 To LD Adapter --- # LDIO NGSIv2 To LD Adapter -***be.vlaanderen.informatievlaanderen.ldes.ldi.NgsiV2ToLdAdapter*** + +***Ldio:NgsiV2ToLdAdapter*** An LDIO wrapper component for the [LDI NGSIv2 To LD building block](../../core/ldi-adapters/ngsiv2-to-ld) diff --git a/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md b/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md index 91e0d6789..1cf6390f4 100644 --- a/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md +++ b/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md @@ -5,7 +5,8 @@ title: RDF Adapter --- # LDIO RDF Adapter -***be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter*** + +***Ldio:RdfAdapter*** An LDIO wrapper component for the [LDI RDF Adapter building block](../../core/ldi-adapters/rdf-adapter) diff --git a/docs/_ldio/ldio-adapters/ldio-rml-adapter.md b/docs/_ldio/ldio-adapters/ldio-rml-adapter.md index 4a5a297e1..d9e9f075a 100644 --- a/docs/_ldio/ldio-adapters/ldio-rml-adapter.md +++ b/docs/_ldio/ldio-adapters/ldio-rml-adapter.md @@ -5,7 +5,8 @@ title: RML Adapter --- # LDIO RML Adapter -***be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter*** + +***Ldio:RmlAdapter*** An LDIO wrapper component for the [LDI RML Adapter building block](../../core/ldi-adapters/rml-adapter) diff --git a/docs/_ldio/ldio-inputs/ldio-amqp-in.md b/docs/_ldio/ldio-inputs/ldio-amqp-in.md index cc4696e44..086cadfd3 100644 --- a/docs/_ldio/ldio-inputs/ldio-amqp-in.md +++ b/docs/_ldio/ldio-inputs/ldio-amqp-in.md @@ -6,7 +6,7 @@ title: AMQP In # LDIO AMQP In -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn*** +***Ldio:AmqpIn*** The LDIO AMQP In listens to messages from an [AMQP 1.0 queue](https://www.amqp.org/resources/specifications). @@ -25,7 +25,7 @@ an [AMQP 1.0 queue](https://www.amqp.org/resources/specifications). ```yaml input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn + name: Ldio:AmqpIn config: remote-url: amqp://localhost:61616 username: artemis diff --git a/docs/_ldio/ldio-inputs/ldio-archive-file-in.md b/docs/_ldio/ldio-inputs/ldio-archive-file-in.md index 554299480..d9b516775 100644 --- a/docs/_ldio/ldio-inputs/ldio-archive-file-in.md +++ b/docs/_ldio/ldio-inputs/ldio-archive-file-in.md @@ -5,7 +5,8 @@ title: Archive File In --- # LDIO File Out -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioArchiveFileIn*** + +***Ldio:ArchiveFileIn*** The LDIO Archive File In is used to read models from files and feed them to the pipeline. Please refer to the [core documentation](../../core/ldi-inputs/file-archiving.md) for more information. diff --git a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md index a977249e6..1f30ecaba 100644 --- a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md +++ b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md @@ -5,7 +5,8 @@ title: HTTP In Poller --- # LDIO HTTP In Poller -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller*** + +***Ldio:HttpInPoller*** The LDIO Http In Poller is a basic Http Poller that will poll a target URL on a specified interval. @@ -24,7 +25,7 @@ Refer to [LDIO Http Requester](../ldio-core) for the config. The Http In Poller supports polling multiple endpoints. Example configuration: ```yaml -name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller +name: Ldio:HttpInPoller config: auth: type: API_KEY diff --git a/docs/_ldio/ldio-inputs/ldio-http-in.md b/docs/_ldio/ldio-inputs/ldio-http-in.md index 3659d7d70..53e664c2c 100644 --- a/docs/_ldio/ldio-inputs/ldio-http-in.md +++ b/docs/_ldio/ldio-inputs/ldio-http-in.md @@ -5,7 +5,8 @@ title: HTTP In --- # LDIO HTTP In -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn*** + +***Ldio:HttpIn*** The LDIO Http In is a basic Http Listener. diff --git a/docs/_ldio/ldio-inputs/ldio-kafka-in.md b/docs/_ldio/ldio-inputs/ldio-kafka-in.md index 8f558f781..80cea40f1 100644 --- a/docs/_ldio/ldio-inputs/ldio-kafka-in.md +++ b/docs/_ldio/ldio-inputs/ldio-kafka-in.md @@ -5,7 +5,8 @@ title: Kafka In --- # LDIO Kafka In -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn*** + +***Ldio:KafkaIn*** The LDIO Kafka In listens to messages from a [kafka topic](https://kafka.apache.org). @@ -31,7 +32,7 @@ Two security protocols are supported: ```yaml outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn + - name: Ldio:KafkaIn config: content-type: application/n-quads topics: quickstart-events @@ -42,7 +43,7 @@ outputs: ```yaml outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn + - name: Ldio:KafkaIn config: content-type: application/n-quads topics: quickstart-events diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md index 40b443ce4..adf8ad2e8 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md @@ -6,7 +6,7 @@ title: LDES Client with Connector # LDIO Ldes Client Connector -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector*** +***Ldio:LdesClientConnector*** An EDC (Eclipse dataspace Connector) LDIO wrapper component for the [LDI LDES Client building block](../../core/ldi-inputs/ldes-client) @@ -45,7 +45,7 @@ This component exposes two endpoints: ```yaml input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector + name: Ldio:LdesClientConnector config: url: http://consumer-connector:29291/public connector-transfer-url: http://consumer-connector:29193/management/v2/transferprocesses diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index 977ba0f86..81a3f8856 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -6,24 +6,25 @@ title: LDES Client # LDIO Ldes Client -***be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient*** +***Ldio:LdesClient*** An LDIO wrapper component for the [LDI LDES Client building block](../../core/ldi-inputs/ldes-client) ## Config -| Property | Description | Required | Default | Example | Supported values | -|:-----------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------|:--------------------------------------|:---------------------------------------------------------------|:--------------------------------------------------------------| -| url | The url of the LDES server | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | -| source-format | The 'Content-Type' that should be requested to the server. | No | application/ld+json | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/) | -| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite' or 'memory' | -| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | -| postgres.url | JDBC url of the Postgres database. | No | N/A | jdbc:postgresql://test.postgres.database.azure.com:5432/sample | String | -| postgres.username | Username used to connect to Postgres database. | No | N/A | myUsername@test | String | -| postgres.password | Password used to connect to Postgres database. | No | N/A | myPassword | String | -| use-version-materialisation | Indicates if the client should return state-objects (true) or version-objects (false). | No | false | true | true or false | -| version-of-property | Property that points to the versionOfPath. | No | http://purl.org/dc/terms/isVersionOf | "http://purl.org/dc/terms/isVersionOf" | true or false | -| restrict-to-members | Indicates that only the statements about the ldes:member should be included, including potential nested blank nodes. If this is enabled, statements about referenced entities are excluded. | No | false | false | true or false | +| Property | Description | Required | Default | Example | Supported values | +|:----------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------|:-------------------------------------|:---------------------------------------------------------------|:--------------------------------------------------------------| +| urls | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | +| source-format | The 'Content-Type' that should be requested to the server. | No | application/ld+json | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/) | +| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite', 'files' or 'memory' | +| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | +| postgres.url | JDBC url of the Postgres database. | No | N/A | jdbc:postgresql://test.postgres.database.azure.com:5432/sample | String | +| postgres.username | Username used to connect to Postgres database. | No | N/A | myUsername@test | String | +| postgres.password | Password used to connect to Postgres database. | No | N/A | myPassword | String | +| timestamp-path | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | +| use-version-materialisation | Indicates if the client should return state-objects (true) or version-objects (false). | No | false | true | true or false | +| version-of-property | Property that points to the versionOfPath. | No | http://purl.org/dc/terms/isVersionOf | "http://purl.org/dc/terms/isVersionOf" | true or false | +| restrict-to-members | Indicates that only the statements about the ldes:member should be included, including potential nested blank nodes. If this is enabled, statements about referenced entities are excluded. | No | false | false | true or false | This component uses the "LDIO Http Requester" to make the HTTP request. Refer to [LDIO Http Requester](../ldio-core) for the config. @@ -32,9 +33,10 @@ Refer to [LDIO Http Requester](../ldio-core) for the config. ```yaml input: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + name: Ldio:LdesClient config: - url: http://localhost:8080/my-ldes + urls: + - http://localhost:8080/my-ldes sourceFormat: text/turtle retries: enabled: true @@ -47,9 +49,10 @@ Refer to [LDIO Http Requester](../ldio-core) for the config. ```yaml input: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + name: Ldio:LdesClient config: - url: http://localhost:8080/my-ldes + urls: + - http://localhost:8080/my-ldes sourceFormat: text/turtle retries: enabled: true diff --git a/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md b/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md index c89c2e0b6..fe9a12316 100644 --- a/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md +++ b/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md @@ -6,7 +6,7 @@ title: Azure Blob Out # LDIO Kafka Out -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdiAzureBlobOut*** +***Ldio:AzureBlobOut*** The LDIO Azure Blob Out writes out messages to an Azure Blob Container. Messages can be written out in any format supported by Apache Jena or in json format. @@ -29,7 +29,7 @@ Messages can be written out in any format supported by Apache Jena or in json fo ```yaml outputs: - - name: "be.vlaanderen.informatievlaanderen.ldes.ldio.LdiAzureBlobOut" + - name: "Ldio:AzureBlobOut" config: lang: "json" storage-account-name: "storageaccount" @@ -42,7 +42,7 @@ outputs: ```yaml outputs: - - name: "be.vlaanderen.informatievlaanderen.ldes.ldio.LdiAzureBlobOut" + - name: "Ldio:AzureBlobOut" config: lang: "n-quads" storage-account-name: "storageaccount" diff --git a/docs/_ldio/ldio-outputs/ldio-console-out.md b/docs/_ldio/ldio-outputs/ldio-console-out.md index 05c630a62..db56a11a6 100644 --- a/docs/_ldio/ldio-outputs/ldio-console-out.md +++ b/docs/_ldio/ldio-outputs/ldio-console-out.md @@ -5,7 +5,8 @@ title: Console Out --- # LDIO Console Out -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioConsoleOut*** + +***Ldio:ConsoleOut*** The LDIO Console Out will output its given model to the console. diff --git a/docs/_ldio/ldio-outputs/ldio-file-out.md b/docs/_ldio/ldio-outputs/ldio-file-out.md index 3fe7a3bed..92e9b835d 100644 --- a/docs/_ldio/ldio-outputs/ldio-file-out.md +++ b/docs/_ldio/ldio-outputs/ldio-file-out.md @@ -5,7 +5,8 @@ title: File Out --- # LDIO File Out -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioFileOut*** + +***Ldio:FileOut*** The LDIO File Out is used to write models to files based on a timestamp path property on the model. Please refer to the [core documentation](../../core/ldi-outputs/file-archiving.md) for more information. diff --git a/docs/_ldio/ldio-outputs/ldio-http-out.md b/docs/_ldio/ldio-outputs/ldio-http-out.md index ef0be626c..106294f27 100644 --- a/docs/_ldio/ldio-outputs/ldio-http-out.md +++ b/docs/_ldio/ldio-outputs/ldio-http-out.md @@ -5,7 +5,8 @@ title: HTTP Out --- # LDIO HTTP Out -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut*** + +***Ldio:HttpOut*** The LDIO HTTP Out is a basic Http Client that will send the given Linked Data model to a target url. diff --git a/docs/_ldio/ldio-outputs/ldio-kafka-out.md b/docs/_ldio/ldio-outputs/ldio-kafka-out.md index e5c052a4d..6d01db3ec 100644 --- a/docs/_ldio/ldio-outputs/ldio-kafka-out.md +++ b/docs/_ldio/ldio-outputs/ldio-kafka-out.md @@ -5,7 +5,8 @@ title: Kafka Out --- # LDIO Kafka Out -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaOut*** + +***Ldio:KafkaOut*** The LDIO Kafka Out sends messages to a [kafka topic](https://kafka.apache.org). Two security protocols are supported: @@ -31,7 +32,7 @@ Two security protocols are supported: ```yaml outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaOut + - name: Ldio:KafkaOut config: bootstrap-servers: localhost:9092 topic: quickstart-events @@ -42,7 +43,7 @@ outputs: ```yaml outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaOut + - name: Ldio:KafkaOut config: bootstrap-servers: localhost:9092 topic: quickstart-events diff --git a/docs/_ldio/ldio-outputs/ldio-repository-materialiser.md b/docs/_ldio/ldio-outputs/ldio-repository-materialiser.md index eda69749d..1be0adbba 100644 --- a/docs/_ldio/ldio-outputs/ldio-repository-materialiser.md +++ b/docs/_ldio/ldio-outputs/ldio-repository-materialiser.md @@ -5,7 +5,8 @@ title: Repository Materialization --- # Repository Materialiser -***be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser*** + +***Ldio:RepositoryMaterialiser*** The repository materialiser is used to materialise an LDES stream into a triplestore. Any triplestore that supports the RDF4J remote repository API can be used. diff --git a/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md b/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md index 2c41150fe..e6a03247c 100644 --- a/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md +++ b/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md @@ -5,7 +5,8 @@ title: GeoJson To WKT Transformer --- # LDIO GeoJson To WKT Transformer -***be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer*** + +***Ldio:GeoJsonToWktTransformer*** An LDIO wrapper component for the [LDI GeoJson to Wkt building block](../../core/ldi-transformers/geojson-to-wkt) diff --git a/docs/_ldio/ldio-transformers/ldio-http-enricher.md b/docs/_ldio/ldio-transformers/ldio-http-enricher.md index 929321008..79a9c6880 100644 --- a/docs/_ldio/ldio-transformers/ldio-http-enricher.md +++ b/docs/_ldio/ldio-transformers/ldio-http-enricher.md @@ -5,23 +5,23 @@ title: Http Enricher Transformer --- # LDIO Http Enricher -***be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher*** + +***Ldio:HttpEnricher*** A transformer which allows to send a GET or POST HTTP request to a dynamic URL provided by the model. The response is converted to linked data and added to the incoming model. ## Config - -| Property | Description | Required | Default | Example | Supported values | -|:------------------------------------|:------------------------------------------------------------------------------------------------------------|:---------|:--------|:-------------------------------------------------------|:---------------------------------------------------------------| -| adapter.name | This transformer requires an [ldio-adapter](../ldio-adapters) to convert the responses to linked data. | Yes | N/A | be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter | Paths of supported LDIO Adapters | -| adapter.config.xxx | Optional config that may be required by the adapter | No | N/A | be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter | Paths of supported LDIO Adapters | -| url-property-path | Path defining the url that needs to be selected on the model for the http request. | Yes | N/A | | Valid property paths | -| header-property-path | Path defining the headers that needs to be selected on the model for the http request. | No | N/A | | Valid property paths | -| body-property-path | Path defining the body that needs to be selected on the model to be added when a POST http request is used. | No | N/A | / | Valid property paths | -| http-method-property-path | Path defining the http method that needs to be selected on the model for the http request. | No | GET | GET | GET or POST | -| auth.xxx retries.xxx rate-limit.xxx | LDIO Http Requester Config | No | N/A | N/A | [LDIO Http Requester Config](../ldio-core/ldio-http-requester) | +| Property | Description | Required | Default | Example | Supported values | +|:------------------------------------|:------------------------------------------------------------------------------------------------------------|:---------|:--------|:----------------------------------------------------|:---------------------------------------------------------------| +| adapter.name | This transformer requires an [ldio-adapter](../ldio-adapters) to convert the responses to linked data. | Yes | N/A | Ldio:RdfAdapter | Paths of supported LDIO Adapters | +| adapter.config.xxx | Optional config that may be required by the adapter | No | N/A | Ldio:RdfAdapter | Paths of supported LDIO Adapters | +| url-property-path | Path defining the url that needs to be selected on the model for the http request. | Yes | N/A | | Valid property paths | +| header-property-path | Path defining the headers that needs to be selected on the model for the http request. | No | N/A | | Valid property paths | +| body-property-path | Path defining the body that needs to be selected on the model to be added when a POST http request is used. | No | N/A | / | Valid property paths | +| http-method-property-path | Path defining the http method that needs to be selected on the model for the http request. | No | GET | GET | GET or POST | +| auth.xxx retries.xxx rate-limit.xxx | LDIO Http Requester Config | No | N/A | N/A | [LDIO Http Requester Config](../ldio-core/ldio-http-requester) | Note that all adapters are supported. When the adapter requires additional config, this can be added as seen below in the example. @@ -31,10 +31,10 @@ This example contains a JsonToLdAdapter which needs "core-context" as config. At the bottom there is also "auth" config provided for the request executor. ```yaml - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher + - name: Ldio:HttpEnricher config: adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter + name: Ldio:JsonToLdAdapter config: core-context: file:///ldio/jsonld/observation.jsonld url-property-path: diff --git a/docs/_ldio/ldio-transformers/ldio-sparql-construct.md b/docs/_ldio/ldio-transformers/ldio-sparql-construct.md index e8b21fe1f..2bb564351 100644 --- a/docs/_ldio/ldio-transformers/ldio-sparql-construct.md +++ b/docs/_ldio/ldio-transformers/ldio-sparql-construct.md @@ -5,7 +5,8 @@ title: SPARQL Construct --- # LDIO SPARQL Construct -***be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer*** + +***Ldio:SparqlConstructTransformer*** An LDIO wrapper component for the [LDI SPARQL Construct building block](../../core/ldi-transformers/sparql-construct) diff --git a/docs/_ldio/ldio-transformers/ldio-version-materializer.md b/docs/_ldio/ldio-transformers/ldio-version-materializer.md index 215b01beb..3ce5a0a51 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-materializer.md +++ b/docs/_ldio/ldio-transformers/ldio-version-materializer.md @@ -5,7 +5,8 @@ title: Version Materializer --- # LDIO Version Materializer -***be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser*** + +***Ldio:VersionMaterialiser*** An LDIO wrapper component for the [LDI Version Materializer building block](../../core/ldi-transformers/version-materializer) diff --git a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md index 9b093cf36..c4031f18d 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md +++ b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md @@ -5,7 +5,8 @@ title: Version Object Creator --- # LDIO Version Object Creator -***be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator*** + +***Ldio:VersionObjectCreator*** An LDIO wrapper component for the [LDI Version Object Creator building block](../../core/ldi-transformers/version-object-creator) diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index f3de2d97e..df2a5b52e 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index 1dfa75503..5e04a67cc 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,9 +6,16 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT file-archiving + + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-common + ${project.version} + + \ No newline at end of file diff --git a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFile.java b/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFile.java index a3e65860d..71ee405ab 100644 --- a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFile.java +++ b/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/ArchiveFile.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import org.apache.jena.rdf.model.Model; import java.nio.file.Path; diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index 9fb5707e6..e268181bf 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index fc515a4d4..15a840f71 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/json-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapterTest.java b/ldi-core/json-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapterTest.java index af77e1590..163fe6ac1 100644 --- a/ldi-core/json-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapterTest.java +++ b/ldi-core/json-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapterTest.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions.UnsupportedMimeTypeException; import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.parser.JenaContextProvider; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.riot.Lang; @@ -22,6 +21,8 @@ import java.util.List; import java.util.stream.Stream; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.*; diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index 2bcb12ca0..7b7a35ac0 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index b2e201c53..7baa0a1c1 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index 25844bc7e..517157721 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,8 +5,14 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT + + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-common + + 4.0.0 tree-node-fetcher \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java index 6611737e2..678acbefe 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import ldes.client.treenodefetcher.domain.valueobjects.ModelResponse; import ldes.client.treenodefetcher.domain.valueobjects.MutabilityStatus; import ldes.client.treenodefetcher.domain.valueobjects.TreeNodeRequest; @@ -18,9 +19,11 @@ public class TreeNodeFetcher { private final RequestExecutor requestExecutor; + private final TimestampExtractor timestampExtractor; - public TreeNodeFetcher(RequestExecutor requestExecutor) { + public TreeNodeFetcher(RequestExecutor requestExecutor, TimestampExtractor timestampExtractor) { this.requestExecutor = requestExecutor; + this.timestampExtractor = timestampExtractor; } public TreeNodeResponse fetchTreeNode(TreeNodeRequest treeNodeRequest) { @@ -45,7 +48,7 @@ public TreeNodeResponse fetchTreeNode(TreeNodeRequest treeNodeRequest) { private TreeNodeResponse createOkResponse(TreeNodeRequest treeNodeRequest, Response response) { final InputStream responseBody = response.getBody().map(ByteArrayInputStream::new).orElseThrow(); final Model model = RDFParser.source(responseBody).forceLang(treeNodeRequest.getLang()).base(treeNodeRequest.getTreeNodeUrl()).toModel(); - final ModelResponse modelResponse = new ModelResponse(model); + final ModelResponse modelResponse = new ModelResponse(model, timestampExtractor); final MutabilityStatus mutabilityStatus = getMutabilityStatus(response); return new TreeNodeResponse(modelResponse.getRelations(), modelResponse.getMembers(), mutabilityStatus); } @@ -67,5 +70,4 @@ private static MutabilityStatus getMutabilityStatus(Response response) { .map(MutabilityStatus::ofHeader) .orElseGet(MutabilityStatus::empty); } - } diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/entities/TreeMember.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/entities/TreeMember.java index e9cdd8b5c..3bc451c03 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/entities/TreeMember.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/entities/TreeMember.java @@ -2,12 +2,16 @@ import org.apache.jena.rdf.model.Model; +import java.time.LocalDateTime; + public class TreeMember { private final String memberId; + private final LocalDateTime createdAt; private final Model model; - public TreeMember(String memberId, Model model) { + public TreeMember(String memberId, LocalDateTime createdAt, Model model) { this.memberId = memberId; + this.createdAt = createdAt; this.model = model; } @@ -18,4 +22,8 @@ public String getMemberId() { public Model getModel() { return model; } + + public LocalDateTime getCreatedAt() { + return createdAt; + } } diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/Constants.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/Constants.java new file mode 100644 index 000000000..6184108df --- /dev/null +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/Constants.java @@ -0,0 +1,17 @@ +package ldes.client.treenodefetcher.domain.valueobjects; + +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.Resource; + +import static org.apache.jena.rdf.model.ResourceFactory.createProperty; + +public class Constants { + public static final Resource ANY_RESOURCE = null; + public static final String W3C_TREE = "https://w3id.org/tree#"; + public static final Property W3ID_TREE_RELATION = createProperty(W3C_TREE, "relation"); + public static final Property W3ID_TREE_MEMBER = createProperty(W3C_TREE, "member"); + public static final Property W3ID_TREE_NODE = createProperty(W3C_TREE, "node"); + + private Constants() { + } +} diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java index 1e8c0bc69..fb78ce57a 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java @@ -1,28 +1,26 @@ package ldes.client.treenodefetcher.domain.valueobjects; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import ldes.client.treenodefetcher.domain.entities.TreeMember; import org.apache.jena.graph.TripleBoundary; import org.apache.jena.rdf.model.*; +import java.time.LocalDateTime; import java.util.Iterator; import java.util.List; import java.util.function.UnaryOperator; import java.util.stream.Stream; -import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +import static ldes.client.treenodefetcher.domain.valueobjects.Constants.*; public class ModelResponse { - - protected static final Resource ANY_RESOURCE = null; - public static final String W3C_TREE = "https://w3id.org/tree#"; - public static final Property W3ID_TREE_RELATION = createProperty(W3C_TREE, "relation"); - public static final Property W3ID_TREE_MEMBER = createProperty(W3C_TREE, "member"); - public static final Property W3ID_TREE_NODE = createProperty(W3C_TREE, "node"); + private TimestampExtractor timestampExtractor; private final ModelExtract modelExtract = new ModelExtract(new StatementTripleBoundary(TripleBoundary.stopNowhere)); private final Model model; - public ModelResponse(Model model) { + public ModelResponse(Model model, TimestampExtractor timestampExtractor) { this.model = model; + this.timestampExtractor = timestampExtractor; } public List getRelations() { @@ -47,8 +45,9 @@ private Stream extractMembers() { private TreeMember processMember(Model treeNodeModel, Statement memberStatement) { final Model memberModel = modelExtract.extract(memberStatement.getObject().asResource(), treeNodeModel); + LocalDateTime createdAt = timestampExtractor.extractTimestamp(memberModel); final String id = memberStatement.getObject().toString(); - return new TreeMember(id, memberModel); + return new TreeMember(id, createdAt, memberModel); } private Stream extractRelations() { diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java index d5c652a2d..8284671e4 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java @@ -1,6 +1,7 @@ package ldes.client.treenodefetcher; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -21,7 +22,7 @@ public class TreeNodeFetcherSteps { @Given("I have a TreeNodeFetcher") public void initializeCalculator() { - treeNodeFetcher = new TreeNodeFetcher(requestExecutorFactory.createNoAuthExecutor()); + treeNodeFetcher = new TreeNodeFetcher(requestExecutorFactory.createNoAuthExecutor(), new TimestampFromCurrentTimeExtractor()); } @When("I create a TreeNodeRequest with Lang {string} and url {string}") diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index c23eb0c1d..1eb62535f 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index a07ee233d..6f3e6eb64 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,12 +5,17 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 tree-node-supplier + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-common + ${project.version} + be.vlaanderen.informatievlaanderen.ldes.client tree-node-fetcher @@ -25,7 +30,6 @@ be.vlaanderen.informatievlaanderen.ldes.ldi version-materialiser ${project.version} - compile diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java index b28087beb..b76171b48 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/MemberSupplierImpl.java @@ -1,22 +1,11 @@ package ldes.client.treenodesupplier; -import ldes.client.treenodesupplier.domain.valueobject.EndOfLdesException; import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; public class MemberSupplierImpl implements MemberSupplier { - private static final long DEFAULT_WAITING_TIME_IN_SECONDS = 100; private final TreeNodeProcessor treeNodeProcessor; private final boolean keepState; - private final Logger logger = LoggerFactory.getLogger(MemberSupplierImpl.class); - private final ExecutorService executorService = Executors.newSingleThreadExecutor(); public MemberSupplierImpl(TreeNodeProcessor treeNodeProcessor, boolean keepState) { this.treeNodeProcessor = treeNodeProcessor; @@ -26,33 +15,13 @@ public MemberSupplierImpl(TreeNodeProcessor treeNodeProcessor, boolean keepState @Override public SuppliedMember get() { - try { - return executorService.submit(treeNodeProcessor::getMember).get(); - } catch (ExecutionException e) { - if (e.getCause() instanceof EndOfLdesException endOfLdesException) { - throw endOfLdesException; - } - throw new ClientInterruptedException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ClientInterruptedException(e); - } + return treeNodeProcessor.getMember(); } @Override public void destroyState() { - try { - executorService.shutdown(); - if (!executorService.awaitTermination(DEFAULT_WAITING_TIME_IN_SECONDS, TimeUnit.SECONDS)) { - logger.warn("ExecutorService wasn't able to finish last task."); - System.exit(0); - } - if (!keepState) { - treeNodeProcessor.destroyState(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ClientInterruptedException(e); + if (!keepState) { + treeNodeProcessor.destroyState(); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java index 7facb0fa2..ea2a4f237 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java @@ -1,6 +1,7 @@ package ldes.client.treenodesupplier; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import ldes.client.treenodefetcher.TreeNodeFetcher; import ldes.client.treenodefetcher.domain.valueobjects.TreeNodeResponse; import ldes.client.treenodesupplier.domain.entities.MemberRecord; @@ -25,14 +26,35 @@ public class TreeNodeProcessor { private MemberRecord memberRecord; public TreeNodeProcessor(LdesMetaData ldesMetaData, StatePersistence statePersistence, - RequestExecutor requestExecutor) { + RequestExecutor requestExecutor, TimestampExtractor timestampExtractor) { this.treeNodeRecordRepository = statePersistence.getTreeNodeRecordRepository(); this.memberRepository = statePersistence.getMemberRepository(); this.requestExecutor = requestExecutor; - this.treeNodeFetcher = new TreeNodeFetcher(requestExecutor); + this.treeNodeFetcher = new TreeNodeFetcher(requestExecutor, timestampExtractor); this.ldesMetaData = ldesMetaData; } + public SuppliedMember getMember() { + savePreviousState(); + if (!treeNodeRecordRepository.containsTreeNodeRecords()) { + initializeTreeNodeRecordRepository(); + } + Optional unprocessedTreeMember = memberRepository.getUnprocessedTreeMember(); + while (unprocessedTreeMember.isEmpty()) { + processTreeNode(); + unprocessedTreeMember = memberRepository.getUnprocessedTreeMember(); + } + MemberRecord treeMember = unprocessedTreeMember.get(); + SuppliedMember suppliedMember = treeMember.createSuppliedMember(); + treeMember.processedMemberRecord(); + memberRecord = treeMember; + return suppliedMember; + } + + public LdesMetaData getLdesMetaData() { + return ldesMetaData; + } + private void processTreeNode() { TreeNodeRecord treeNodeRecord = treeNodeRecordRepository .getOneTreeNodeRecordWithStatus(TreeNodeStatus.NOT_VISITED).orElseGet( @@ -49,11 +71,10 @@ private void processTreeNode() { .filter(treeNodeId -> !treeNodeRecordRepository.existsById(treeNodeId)) .map(TreeNodeRecord::new) .forEach(treeNodeRecordRepository::saveTreeNodeRecord); - treeNodeResponse.getMembers() + memberRepository.saveTreeMembers(treeNodeResponse.getMembers() .stream() - .map(treeMember -> new MemberRecord(treeMember.getMemberId(), treeMember.getModel())) - .filter(member -> !memberRepository.isProcessed(member)) - .forEach(memberRepository::saveTreeMember); + .map(treeMember -> new MemberRecord(treeMember.getMemberId(), treeMember.getModel(), treeMember.getCreatedAt())) + .filter(member -> !memberRepository.isProcessed(member))); } @@ -69,27 +90,13 @@ private void waitUntilNextVisit(TreeNodeRecord treeNodeRecord) { } } - public SuppliedMember getMember() { - savePreviousState(); - if (!treeNodeRecordRepository.containsTreeNodeRecords()) { - initializeTreeNodeRecordRepository(); - } - Optional unprocessedTreeMember = memberRepository.getUnprocessedTreeMember(); - while (unprocessedTreeMember.isEmpty()) { - processTreeNode(); - unprocessedTreeMember = memberRepository.getUnprocessedTreeMember(); - } - MemberRecord treeMember = unprocessedTreeMember.get(); - SuppliedMember suppliedMember = treeMember.createSuppliedMember(); - treeMember.processedMemberRecord(); - memberRecord = treeMember; - return suppliedMember; - } - private void initializeTreeNodeRecordRepository() { - StartingTreeNode start = new StartingTreeNodeSupplier(requestExecutor) - .getStart(ldesMetaData.getStartingNodeUrl(), ldesMetaData.getLang()); - treeNodeRecordRepository.saveTreeNodeRecord(new TreeNodeRecord(start.getStartingNodeUrl())); + ldesMetaData.getStartingNodeUrls() + .stream() + .map(startingNode -> new StartingTreeNodeSupplier(requestExecutor) + .getStart(startingNode, ldesMetaData.getLang())) + .map(start -> new TreeNodeRecord(start.getStartingNodeUrl())) + .forEach(treeNodeRecordRepository::saveTreeNodeRecord); } private void savePreviousState() { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java index 5472c200f..92cfca150 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java @@ -4,22 +4,27 @@ import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; import org.apache.jena.rdf.model.Model; +import java.time.LocalDateTime; import java.util.Objects; -public class MemberRecord { +import com.sun.istack.NotNull; + +public class MemberRecord implements Comparable{ private final String memberId; private MemberStatus memberStatus; + private LocalDateTime createdAt; private Model model; - public MemberRecord(String memberId, Model model) { - this(memberId, model, MemberStatus.UNPROCESSED); + public MemberRecord(String memberId, Model model, LocalDateTime createdAt) { + this(memberId, model, MemberStatus.UNPROCESSED, createdAt); } - public MemberRecord(String memberId, Model model, MemberStatus memberStatus) { + public MemberRecord(String memberId, Model model, MemberStatus memberStatus, LocalDateTime createdAt) { this.memberId = memberId; this.model = model; this.memberStatus = memberStatus; + this.createdAt = createdAt; } public void processedMemberRecord() { @@ -56,4 +61,13 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(memberId); } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + @Override + public int compareTo(@NotNull MemberRecord member) { + return getCreatedAt().compareTo(member.getCreatedAt()); + } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesMetaData.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesMetaData.java index 5a61ab8f0..65b82c5ef 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesMetaData.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesMetaData.java @@ -3,17 +3,26 @@ import ldes.client.treenodefetcher.domain.valueobjects.TreeNodeRequest; import org.apache.jena.riot.Lang; +import java.util.List; + public class LdesMetaData { - private final String startingNodeUrl; + private final List startingNodeUrls; private final Lang lang; - public LdesMetaData(String startingNodeUrl, Lang lang) { - this.startingNodeUrl = startingNodeUrl; + public LdesMetaData(List startingNodeUrls, Lang lang) { + this.startingNodeUrls = startingNodeUrls; this.lang = lang; } + public List getStartingNodeUrls() { + return startingNodeUrls; + } + public String getStartingNodeUrl() { - return startingNodeUrl; + if (startingNodeUrls.isEmpty()) { + throw new IllegalArgumentException("No starting nodes defined for LDES Client."); + } + return startingNodeUrls.get(0); } public TreeNodeRequest createRequest(String treeNodeUrl) { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java index 7b81e45e3..e1a751151 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java @@ -3,6 +3,7 @@ import ldes.client.treenodesupplier.domain.entities.MemberRecord; import java.util.Optional; +import java.util.stream.Stream; public interface MemberRepository { @@ -10,6 +11,8 @@ public interface MemberRepository { boolean isProcessed(MemberRecord member); + void saveTreeMembers(Stream treeMemberStream); + void saveTreeMember(MemberRecord treeMember); void destroyState(); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/FileBasedMemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/FileBasedMemberRepository.java index cbd044ef9..9b4fe6fb2 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/FileBasedMemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/FileBasedMemberRepository.java @@ -5,9 +5,11 @@ import ldes.client.treenodesupplier.repository.MemberRepository; import ldes.client.treenodesupplier.repository.filebased.mapper.MemberRecordMapper; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; public class FileBasedMemberRepository implements MemberRepository { public static final String UNPROCESSED_MEMBERS = "membersUnprocessed.txt"; @@ -26,7 +28,7 @@ public Optional getUnprocessedTreeMember() { return fileManager .getRecords(UNPROCESSED_MEMBERS) .map(mapper::toMemberRecord) - .findFirst(); + .min(Comparator.comparing(MemberRecord::getCreatedAt)); } @Override @@ -37,6 +39,11 @@ public boolean isProcessed(MemberRecord member) { .anyMatch(memberRecord -> memberRecord.equals(member)); } + @Override + public void saveTreeMembers(Stream treeMemberStream) { + treeMemberStream.forEach(this::saveTreeMember); + } + @Override public void saveTreeMember(MemberRecord treeMember) { if (treeMember.getMemberStatus() == MemberStatus.PROCESSED) { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/mapper/MemberRecordMapper.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/mapper/MemberRecordMapper.java index b837d7fd3..06c2c74a9 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/mapper/MemberRecordMapper.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/filebased/mapper/MemberRecordMapper.java @@ -7,16 +7,24 @@ import org.apache.jena.riot.RDFParserBuilder; import org.apache.jena.riot.RDFWriter; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + public class MemberRecordMapper { public static final String DELIMITER = ";;;"; + private DateTimeFormatter dateTimeFormatter; + + public MemberRecordMapper() { + dateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME; + } public String fromMemberRecord(MemberRecord memberRecord) { if (memberRecord.getModel() != null) { - return String.join(DELIMITER, memberRecord.getMemberId(), memberRecord.getMemberStatus().toString(), + return String.join(DELIMITER, memberRecord.getMemberId(), memberRecord.getMemberStatus().toString(), dateTimeFormatter.format(memberRecord.getCreatedAt()), RDFWriter.source(memberRecord.getModel()).lang(Lang.NQUADS).asString().replace("\n", "#NEWLINE")); } else { - return String.join(DELIMITER, memberRecord.getMemberId(), memberRecord.getMemberStatus().toString()); + return String.join(DELIMITER, memberRecord.getMemberId(), memberRecord.getMemberStatus().toString(), dateTimeFormatter.format(memberRecord.getCreatedAt())); } } @@ -24,10 +32,10 @@ public String fromMemberRecord(MemberRecord memberRecord) { public MemberRecord toMemberRecord(String line) { String[] parts = line.split(DELIMITER); Model model = null; - if (parts.length > 2) { - model = RDFParserBuilder.create().fromString(parts[2].replace("#NEWLINE", "\n")).lang(Lang.NQUADS) + if (parts.length > 3) { + model = RDFParserBuilder.create().fromString(parts[3].replace("#NEWLINE", "\n")).lang(Lang.NQUADS) .toModel(); } - return new MemberRecord(parts[0], model, MemberStatus.valueOf(parts[1])); + return new MemberRecord(parts[0], model, MemberStatus.valueOf(parts[1]), LocalDateTime.from(dateTimeFormatter.parse(parts[2]))); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java index 2bdd3f97e..347ce75a4 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java @@ -4,20 +4,20 @@ import ldes.client.treenodesupplier.domain.valueobject.MemberStatus; import ldes.client.treenodesupplier.repository.MemberRepository; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; +import java.util.stream.Stream; public class InMemoryMemberRepository implements MemberRepository { - private List unprocessed = new ArrayList<>(); + private Queue unprocessed = new PriorityQueue<>(); private List processed = new ArrayList<>(); + @Override public Optional getUnprocessedTreeMember() { if (unprocessed.isEmpty()) { return Optional.empty(); } - return Optional.of(unprocessed.remove(0)); + return Optional.of(unprocessed.poll()); } @Override @@ -25,19 +25,24 @@ public boolean isProcessed(MemberRecord member) { return processed.contains(member); } + @Override + public void saveTreeMembers(Stream treeMemberStream) { + treeMemberStream.forEach(this::saveTreeMember); + } + @Override public void saveTreeMember(MemberRecord treeMember) { if (treeMember.getMemberStatus() == MemberStatus.PROCESSED) { processed.add(treeMember); } else if (treeMember.getMemberStatus() == MemberStatus.UNPROCESSED) { - unprocessed.add(treeMember); + unprocessed.offer(treeMember); } } @Override public void destroyState() { processed = new ArrayList<>(); - unprocessed = new ArrayList<>(); + unprocessed = new PriorityQueue<>(); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/MemberRecordEntity.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/MemberRecordEntity.java index 109806b75..17d0d588f 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/MemberRecordEntity.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/MemberRecordEntity.java @@ -7,13 +7,15 @@ import org.apache.jena.riot.RDFParserBuilder; import org.apache.jena.riot.RDFWriter; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.NamedQuery; +import java.time.LocalDateTime; + +import javax.persistence.*; @Entity -@NamedQuery(name = "Member.getByMemberStatus", query = "SELECT m FROM MemberRecordEntity m WHERE m.memberStatus = :memberStatus") +@Table(indexes = { + @Index(name = "fn_index", columnList = "createdAt") +}) +@NamedQuery(name = "Member.getFirstByMemberStatus", query = "SELECT m FROM MemberRecordEntity m WHERE m.memberStatus = :memberStatus ORDER BY m.createdAt") @NamedQuery(name = "Member.countByMemberStatusAndId", query = "SELECT COUNT(m) FROM MemberRecordEntity m WHERE m.memberStatus = :memberStatus and m.id = :id") public class MemberRecordEntity { @@ -22,6 +24,7 @@ public class MemberRecordEntity { private String id; private MemberStatus memberStatus; + private LocalDateTime createdAt; @Column(name = "model", columnDefinition = "text", length = 10485760) private String modelAsString; @@ -29,20 +32,22 @@ public class MemberRecordEntity { public MemberRecordEntity() { } - public MemberRecordEntity(String id, MemberStatus memberStatus, String modelAsString) { + public MemberRecordEntity(String id, MemberStatus memberStatus, LocalDateTime dateCreated, String modelAsString) { this.id = id; this.memberStatus = memberStatus; + this.createdAt = dateCreated; this.modelAsString = modelAsString; } public static MemberRecordEntity fromMemberRecord(MemberRecord treeMember) { final Model model = treeMember.getModel(); final String localModalString = model != null ? RDFWriter.source(model).lang(Lang.NQUADS).asString() : null; - return new MemberRecordEntity(treeMember.getMemberId(), treeMember.getMemberStatus(), localModalString); + return new MemberRecordEntity(treeMember.getMemberId(), treeMember.getMemberStatus(), treeMember.getCreatedAt(), localModalString); } public MemberRecord toMemberRecord() { final Model model = RDFParserBuilder.create().fromString(modelAsString).lang(Lang.NQUADS).toModel(); - return new MemberRecord(id, model, memberStatus); + return new MemberRecord(id, model, memberStatus, createdAt); } + } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java index 4c85be7a2..24e479015 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java @@ -5,6 +5,7 @@ import ldes.client.treenodesupplier.repository.MemberRepository; import java.util.Optional; +import java.util.stream.Stream; import javax.persistence.EntityManager; @@ -23,7 +24,7 @@ public SqlMemberRepository(String instanceName, EntityManagerFactory entityManag public Optional getUnprocessedTreeMember() { return entityManager - .createNamedQuery("Member.getByMemberStatus", MemberRecordEntity.class) + .createNamedQuery("Member.getFirstByMemberStatus", MemberRecordEntity.class) .setParameter("memberStatus", MemberStatus.UNPROCESSED) .getResultStream() .map(MemberRecordEntity::toMemberRecord) @@ -40,6 +41,14 @@ public boolean isProcessed(MemberRecord member) { .getSingleResult() > 0; } + @Override + public void saveTreeMembers(Stream treeMemberStream) { + entityManager.getTransaction().begin(); + treeMemberStream.map(MemberRecordEntity::fromMemberRecord) + .forEach(entityManager::merge); + entityManager.getTransaction().commit(); + } + @Override public void saveTreeMember(MemberRecord treeMember) { MemberRecordEntity memberRecordEntity = MemberRecordEntity.fromMemberRecord(treeMember); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java index 6a1651815..685d1089b 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/PerformanceTest.java @@ -1,5 +1,8 @@ package ldes.client.performance; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.extension.responsetemplating.ResponseTemplateTransformer; import ldes.client.performance.csvwriter.CsvFile; import ldes.client.treenodesupplier.TreeNodeProcessor; import org.junit.jupiter.api.*; @@ -8,10 +11,6 @@ import java.time.temporal.ChronoUnit; import java.util.List; -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.core.WireMockConfiguration; -import com.github.tomakehurst.wiremock.extension.responsetemplating.ResponseTemplateTransformer; - import static org.apache.commons.io.FilenameUtils.separatorsToSystem; /** @@ -106,7 +105,7 @@ private void testRunner(String fileName, int testSize, List scenar private void runTest(TestScenario test, CsvFile csvFile, int testSize) { final TreeNodeProcessor treeNodeProcessor = treeNodeProcessorFactory - .createTreeNodeProcessor(test.getPersistenceStrategy(), test.getStartingEndpoint()); + .createTreeNodeProcessor(test.getPersistenceStrategy(), List.of(test.getStartingEndpoint())); LocalDateTime lastInterval = LocalDateTime.now(); for (int i = 1; i <= testSize; i++) { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java index da0e8a767..6d74c7008 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java @@ -2,6 +2,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; import ldes.client.treenodesupplier.TreeNodeProcessor; import ldes.client.treenodesupplier.domain.services.MemberRepositoryFactory; import ldes.client.treenodesupplier.domain.services.TreeNodeRecordRepositoryFactory; @@ -14,13 +16,14 @@ import org.apache.jena.riot.Lang; import org.testcontainers.containers.PostgreSQLContainer; +import java.util.List; import java.util.Map; class TreeNodeProcessorFactory { private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); - TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersistenceStrategy, String url) { + TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersistenceStrategy, List url) { final LdesMetaData ldesMetaData = new LdesMetaData(url, Lang.TURTLE); final StatePersistence statePersistence = switch (statePersistenceStrategy) { case MEMORY -> createInMemoryStatePersistence(); @@ -29,7 +32,8 @@ TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersiste case POSTGRES -> createPostgresPersistence(); }; final RequestExecutor requestExecutor = requestExecutorFactory.createNoAuthExecutor(); - return new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor); + final TimestampExtractor timestampExtractor = new TimestampFromCurrentTimeExtractor(); + return new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor); } private PostgreSQLContainer startPostgresContainer() { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java index ea447b0d4..b51d60b26 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java @@ -1,6 +1,9 @@ package ldes.client.treenodesupplier; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; +import io.cucumber.java.Before; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -14,13 +17,15 @@ import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFDataMgr; +import org.junit.After; import org.testcontainers.containers.PostgreSQLContainer; import java.io.StringWriter; +import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +import static org.junit.jupiter.api.Assertions.*; public class MemberSupplierSteps { private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); @@ -36,6 +41,21 @@ public class MemberSupplierSteps { private final MemberSupplier[] memberSuppliers = new MemberSupplier[2]; private final SuppliedMember[] suppliedMembers = new SuppliedMember[2]; + private String timestampPath; + + @Before + public void setup() { + timestampPath = ""; + } + + @After + public void teardown() { + if (postgreSQLContainer != null) { + postgreSQLContainer.stop(); + postgreSQLContainer = null; + } + } + @When("I request one member from the MemberSupplier") public void iRequestOneMemberFromTheMemberSupplier() { suppliedMember = memberSupplier.get(); @@ -53,15 +73,25 @@ public void statusForTreeNodeRecordWithIdentifier(String treeNodeStatus, String @Given("A starting url {string}") public void aStartingUrl(String url) { - ldesMetaData = new LdesMetaData(url, - Lang.JSONLD); + ldesMetaData = new LdesMetaData(List.of(url), Lang.JSONLD); + } + + @Given("^Starting urls$") + public void startingUrls(List urls) { + ldesMetaData = new LdesMetaData(urls, Lang.TURTLE); + } + + @Given("I set a timestamp path {string}") + public void setTimestampPath(String timestampPath) { + this.timestampPath = timestampPath; } @When("I create a Processor") public void iCreateAProcessor() { treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, new StatePersistence(memberRepository, treeNodeRecordRepository), - requestExecutorFactory.createNoAuthExecutor()); + requestExecutorFactory.createNoAuthExecutor(), + timestampPath.isEmpty() ? new TimestampFromCurrentTimeExtractor() : new TimestampFromPathExtractor(createProperty(timestampPath))); } @Then("Member {string} is processed") @@ -89,6 +119,12 @@ public StatePersistence aSqliteStatePersistenceStrategy() { @And("a StatePersistenceStrategy POSTGRES") public StatePersistence aPostgresStatePersistenceStrategy() { + postgreSQLContainer = new PostgreSQLContainer("postgres:11.1") + .withDatabaseName("integration-test-client-persistence") + .withUsername("sa") + .withPassword("sa"); + postgreSQLContainer.start(); + PostgresProperties postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(), postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false); memberRepository = MemberRepositoryFactory.getMemberRepository(StatePersistenceStrategy.POSTGRES, @@ -131,23 +167,6 @@ public void iCreateAMemberSupplierWithoutState() { memberSupplier = new MemberSupplierImpl(treeNodeProcessor, false); } - @And("Postgres TestContainer is started") - public void postgresTestcontainerIsStarted() { - postgreSQLContainer = new PostgreSQLContainer("postgres:11.1") - .withDatabaseName("integration-test-client-persistence") - .withUsername("sa") - .withPassword("sa"); - postgreSQLContainer.start(); - } - - @And("Postgres TestContainer is stopped") - public void postgresTestContainerIsStopped() { - if (postgreSQLContainer != null) { - postgreSQLContainer.stop(); - postgreSQLContainer = null; - } - } - private StatePersistence defineStatePersistence(String persistenceStrategy) { return switch (persistenceStrategy) { case "FILE" -> aFileStatePersistenceStrategy(); @@ -160,12 +179,16 @@ private StatePersistence defineStatePersistence(String persistenceStrategy) { @And("a {word} MemberSupplier and a {word} MemberSupplier") public void aStatePersistenceStrategyProcessorAndAStatePersistenceStrategyProcessor(String arg0, String arg1) { + timestampPath = "http://www.w3.org/ns/prov#generatedAtTime"; + memberSuppliers[0] = new MemberSupplierImpl(new TreeNodeProcessor(ldesMetaData, defineStatePersistence(arg0), - requestExecutorFactory.createNoAuthExecutor()), false); + requestExecutorFactory.createNoAuthExecutor(), + new TimestampFromPathExtractor(createProperty(timestampPath))), false); memberSuppliers[1] = new MemberSupplierImpl(new TreeNodeProcessor(ldesMetaData, - defineStatePersistence(arg0), - requestExecutorFactory.createNoAuthExecutor()), false); + defineStatePersistence(arg1), + requestExecutorFactory.createNoAuthExecutor(), + new TimestampFromPathExtractor(createProperty(timestampPath))), false); } @When("I request one member from the MemberSuppliers") @@ -176,8 +199,12 @@ public void iRequestOneMemberFromTheMemberSuppliers() { @Then("Member {string} is processed in both MemberSuppliers") public void memberIsProcessedInBothMemberSuppliers(String memberId) { - assertTrue(toString(suppliedMembers[0].getModel(), Lang.JSONLD).contains(memberId)); - assertTrue(toString(suppliedMembers[1].getModel(), Lang.JSONLD).contains(memberId)); + var property = createProperty("http://purl.org/dc/terms/created"); + + var supplier0MemberId = suppliedMembers[0].getModel().listSubjectsWithProperty(property).nextResource().asNode().toString(); + assertEquals(memberId, supplier0MemberId); + var supplier1MemberId = suppliedMembers[1].getModel().listSubjectsWithProperty(property).nextResource().asNode().toString(); + assertEquals(memberId, supplier1MemberId); } @Then("MemberSuppliers are destroyed") diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-1-fragment.ttl b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-1-fragment.ttl new file mode 100644 index 000000000..9b7a46bf3 --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-1-fragment.ttl @@ -0,0 +1,38 @@ +@prefix tree: . +@prefix rdf: . +@prefix ldes: . +@prefix schema: . +@prefix terms: . +@prefix prov: . + + + rdf:type tree:Node ; + terms:isPartOf ; + tree:relation [ rdf:type tree:Relation ; + tree:node ] . + + + + rdf:type ldes:EventStream ; + tree:member , , , + . + + + a schema:Person ; + schema:name "Jane Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.000Z"^^ . + + + a schema:Person ; + schema:name "Jack Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.001Z"^^ . + + + a schema:Person ; + schema:name "Joe Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.002Z"^^ . + + + a schema:Person ; + schema:name "James Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.003Z"^^ . \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-2-fragment.ttl b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-2-fragment.ttl new file mode 100644 index 000000000..e0d69588c --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-2-fragment.ttl @@ -0,0 +1,36 @@ +@prefix tree: . +@prefix rdf: . +@prefix ldes: . +@prefix schema: . +@prefix terms: . +@prefix prov: . + + + rdf:type tree:Node ; + terms:isPartOf . + + + + rdf:type ldes:EventStream ; + tree:member , , , + . + + + a schema:Person ; + schema:name "Jane Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.000Z"^^ . + + + a schema:Person ; + schema:name "Jack Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.001Z"^^ . + + + a schema:Person ; + schema:name "Joe Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.002Z"^^ . + + + a schema:Person ; + schema:name "James Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.003Z"^^ . diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-3-fragment.ttl b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-3-fragment.ttl new file mode 100644 index 000000000..f7582092d --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/__files/items-group-3-fragment.ttl @@ -0,0 +1,36 @@ +@prefix tree: . +@prefix rdf: . +@prefix ldes: . +@prefix schema: . +@prefix terms: . +@prefix prov: . + + + rdf:type tree:Node ; + terms:isPartOf . + + + + rdf:type ldes:EventStream ; + tree:member , , , + . + + + a schema:Person ; + schema:name "Jane Bear" ; + prov:generatedAtTime "2024-02-07T00:00:00.000Z"^^ . + + + a schema:Person ; + schema:name "Jack Bear" ; + prov:generatedAtTime "2024-02-07T00:00:00.001Z"^^ . + + + a schema:Person ; + schema:name "Joe Bear" ; + prov:generatedAtTime "2024-02-07T00:00:00.002Z"^^ . + + + a schema:Person ; + schema:name "James Bear" ; + prov:generatedAtTime "2024-02-07T00:00:00.003Z"^^ . \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/member-supplier.feature b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/member-supplier.feature index f736c3288..c83013088 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/member-supplier.feature +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/member-supplier.feature @@ -4,7 +4,6 @@ Feature: MemberSupplier Scenario Outline: Obtaining the members from first three fragments including the starting node Given A starting url "http://localhost:10101/302-redirects-to-first-node" - And Postgres TestContainer is started And a StatePersistenceStrategy And The TreeNode is not processed: "http://localhost:10101/200-first-tree-node" When I create a Processor @@ -22,7 +21,6 @@ Feature: MemberSupplier Then Status "MUTABLE_AND_ACTIVE" for TreeNodeRecord with identifier: "http://localhost:10101/200-second-tree-node" Then Member "https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/3" is processed Then MemberSupplier is destroyed - And Postgres TestContainer is stopped Examples: | statePersistenceStrategy | @@ -30,3 +28,68 @@ Feature: MemberSupplier | SQLITE | | FILE | | POSTGRES | + + + + Scenario Outline: Obtaining the members from first three fragments including the starting node in order + Given A starting url "http://localhost:10101/200-tree-node-not-ordered" + And a StatePersistenceStrategy + And The TreeNode is not processed: "http://localhost:10101/200-tree-node-not-ordered" + And I set a timestamp path "http://www.w3.org/ns/prov#generatedAtTime" + When I create a Processor + When I create a MemberSupplier without state + When I request one member from the MemberSupplier + Then Member "https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/2" is processed + When I request one member from the MemberSupplier + Then Member "https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/1" is processed + Then MemberSupplier is destroyed + + Examples: + | statePersistenceStrategy | + | MEMORY | + | SQLITE | + | FILE | + | POSTGRES | + + Scenario Outline: Obtaining the members from multiple endpoints + Given Starting urls + | http://localhost:10101/items/grouped?group=1 | + | http://localhost:10101/items/grouped?group=2 | + And a StatePersistenceStrategy + And I set a timestamp path "http://www.w3.org/ns/prov#generatedAtTime" + And The TreeNode is not processed: "http://localhost:10101/items/grouped?group=1" + And The TreeNode is not processed: "http://localhost:10101/items/grouped?group=2" + When I create a Processor + When I create a MemberSupplier without state + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/1" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/2" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/3" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/4" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/5" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/6" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/7" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/8" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/9" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/10" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/11" is processed + When I request one member from the MemberSupplier + Then Member "http://localhost:10101/items/12" is processed + Then MemberSupplier is destroyed + + Examples: + | statePersistenceStrategy | + | MEMORY | + | SQLITE | + | FILE | + | POSTGRES | \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/multi-member-supplier.feature b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/multi-member-supplier.feature index 447c24649..ba1b5ce50 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/multi-member-supplier.feature +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/multi-member-supplier.feature @@ -4,7 +4,6 @@ Feature: MemberSupplier Scenario Outline: Obtaining the members from first three fragments including the starting node in two memberSuppliers Given A starting url "http://localhost:10101/302-redirects-to-first-node" - And Postgres TestContainer is started And a MemberSupplier and a MemberSupplier And The TreeNode is not processed: "http://localhost:10101/200-first-tree-node" When I request one member from the MemberSuppliers @@ -14,7 +13,6 @@ Feature: MemberSupplier When I request one member from the MemberSuppliers Then Member "https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/3" is processed in both MemberSuppliers Then MemberSuppliers are destroyed - And Postgres TestContainer is stopped Examples: | statePersistenceStrategy1 | statePersistenceStrategy2 | diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/restart-client.feature b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/restart-client.feature index afe002640..ab94603ec 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/restart-client.feature +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/features/restart-client.feature @@ -4,7 +4,6 @@ Feature: Restart MemberSupplier Scenario Outline: Obtaining the members from first three fragments including the starting node Given A starting url "http://localhost:10101/302-redirects-to-first-node" - And Postgres TestContainer is started And a StatePersistenceStrategy And The TreeNode is not processed: "http://localhost:10101/200-first-tree-node" When I create a Processor @@ -28,7 +27,6 @@ Feature: Restart MemberSupplier Then Status "MUTABLE_AND_ACTIVE" for TreeNodeRecord with identifier: "http://localhost:10101/200-second-tree-node" Then Member "https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/3" is processed Then MemberSupplier is destroyed - And Postgres TestContainer is stopped Examples: | statePersistenceStrategy | diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-second-tree-node.json b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-second-tree-node.json index 61a710d0c..35303b340 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-second-tree-node.json +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-second-tree-node.json @@ -36,7 +36,7 @@ }, "response": { "status": 200, - "body": "{\"@context\":[\"https://private-api.gipod.beta-vlaanderen.be/api/v1/context/gipod.jsonld\"],\"@id\":\"http://localhost:10101/200-second-tree-node\",\"@type\":\"Node\",\"viewOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"collectionInfo\":{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"@type\":\"EventStream\",\"shape\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances/shape\",\"timestampPath\":\"prov:generatedAtTime\",\"versionOfPath\":\"dct:isVersionOf\"},\"tree:relation\":[{\"tree:node\":\"http://localhost:10101/200-first-tree-node\",\"@type\":\"tree:LessThanRelation\",\"tree:path\":\"prov:generatedAtTime\",\"tree:value\":\"2020-12-28T09:36:09.72Z\"}],\"items\":[{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/2\",\"@type\":\"MobilityHindrance\",\"gipodId\":10228787,\"identifier\":[{\"@type\":\"Identifier\",\"Identifier.identifier\":{\"value\":\"10228787\",\"type\":\"gipodId\"},\"assignedByName\":\"https://gipod.vlaanderen.be\"}],\"isConsequenceOf\":[{\"@id\":\"/api/v1/events/3773285\",\"@type\":\"Event\",\"gipodId\":3773285}],\"description\":\"Eetwagen\",\"owner\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"contactOrganisation\":[],\"zone\":[{\"@id\":\"/api/v1/mobility-hindrances/10228787/zones/6dd8d314-9415-4103-bb06-c7c9fbf3dc19\",\"@type\":\"Zone\",\"consequence\":[{\"@id\":\"/api/v1/taxonomies/mobility-hindrance/consequencetypes/c53813ab-814f-4ff4-8a87-6934c72e175f\",\"prefLabel\":\"Geen doorgang voor gemotoriseerd verkeer\"}],\"geometry\":{\"@type\":\"Geometry\",\"wkt\":\" POLYGON ((109802.90525490482 189774.7184551563, 109801.10560758458 189779.91360264365, 109803.47330090578 189780.72935110983, 109805.27295029224 189775.5342052644, 109807.07260330272 189770.33905491605, 109804.70490584878 189769.52330316138, 109802.90525490482 189774.7184551563))\"},\"zoneType\":{\"@id\":\"/api/v1/taxonomies/zonetypes/0fb72ef7-6ac9-4a70-b295-a30ea215d250\",\"prefLabel\":\"HinderZone\"}}],\"period\":[{\"@type\":\"Period\",\"start\":\"2021-01-31T23:00:00Z\",\"end\":\"2021-02-03T22:30:00Z\"}],\"timeSchedule\":null,\"permittedBy\":[],\"status\":{\"@id\":\"/api/v1/taxonomies/statuses/a411c53e-db33-436a-9bb9-d62d535b661d\",\"prefLabel\":\"Onbekend\"},\"generatedAtTime\":\"2020-12-28T09:37:18.577Z\",\"eventName\":\"MobilityHindranceWasImportedFromLegacy\",\"isVersionOf\":\"/api/v1/mobility-hindrances/10228787\",\"memberOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"lastModifiedOn\":\"2020-12-07T15:10:22.18Z\",\"lastModifiedBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"createdOn\":\"2020-12-07T15:10:22.18Z\",\"createdBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"}},{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/3\",\"@type\":\"MobilityHindrance\",\"gipodId\":10228787,\"identifier\":[{\"@type\":\"Identifier\",\"Identifier.identifier\":{\"value\":\"10228787\",\"type\":\"gipodId\"},\"assignedByName\":\"https://gipod.vlaanderen.be\"}],\"isConsequenceOf\":[{\"@id\":\"/api/v1/events/3773285\",\"@type\":\"Event\",\"gipodId\":3773285}],\"description\":\"Eetwagen\",\"owner\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"contactOrganisation\":[],\"zone\":[{\"@id\":\"/api/v1/mobility-hindrances/10228787/zones/6dd8d314-9415-4103-bb06-c7c9fbf3dc19\",\"@type\":\"Zone\",\"consequence\":[{\"@id\":\"/api/v1/taxonomies/mobility-hindrance/consequencetypes/c53813ab-814f-4ff4-8a87-6934c72e175f\",\"prefLabel\":\"Geen doorgang voor gemotoriseerd verkeer\"}],\"geometry\":{\"@type\":\"Geometry\",\"wkt\":\" POLYGON ((109802.90525490482 189774.7184551563, 109801.10560758458 189779.91360264365, 109803.47330090578 189780.72935110983, 109805.27295029224 189775.5342052644, 109807.07260330272 189770.33905491605, 109804.70490584878 189769.52330316138, 109802.90525490482 189774.7184551563))\"},\"zoneType\":{\"@id\":\"/api/v1/taxonomies/zonetypes/0fb72ef7-6ac9-4a70-b295-a30ea215d250\",\"prefLabel\":\"HinderZone\"}}],\"period\":[{\"@type\":\"Period\",\"start\":\"2021-01-31T23:00:00Z\",\"end\":\"2021-02-03T22:30:00Z\"}],\"timeSchedule\":null,\"permittedBy\":[],\"status\":{\"@id\":\"/api/v1/taxonomies/statuses/a411c53e-db33-436a-9bb9-d62d535b661d\",\"prefLabel\":\"Onbekend\"},\"generatedAtTime\":\"2020-12-28T09:37:18.577Z\",\"eventName\":\"MobilityHindranceWasImportedFromLegacy\",\"isVersionOf\":\"/api/v1/mobility-hindrances/10228787\",\"memberOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"lastModifiedOn\":\"2020-12-07T15:10:22.18Z\",\"lastModifiedBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"createdOn\":\"2020-12-07T15:10:22.18Z\",\"createdBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"}}]}", + "body": "{\"@context\":[\"https://private-api.gipod.beta-vlaanderen.be/api/v1/context/gipod.jsonld\"],\"@id\":\"http://localhost:10101/200-second-tree-node\",\"@type\":\"Node\",\"viewOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"collectionInfo\":{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"@type\":\"EventStream\",\"shape\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances/shape\",\"timestampPath\":\"prov:generatedAtTime\",\"versionOfPath\":\"dct:isVersionOf\"},\"tree:relation\":[{\"tree:node\":\"http://localhost:10101/200-first-tree-node\",\"@type\":\"tree:LessThanRelation\",\"tree:path\":\"prov:generatedAtTime\",\"tree:value\":\"2020-12-28T09:36:09.72Z\"}],\"items\":[{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/2\",\"@type\":\"MobilityHindrance\",\"gipodId\":10228787,\"identifier\":[{\"@type\":\"Identifier\",\"Identifier.identifier\":{\"value\":\"10228787\",\"type\":\"gipodId\"},\"assignedByName\":\"https://gipod.vlaanderen.be\"}],\"isConsequenceOf\":[{\"@id\":\"/api/v1/events/3773285\",\"@type\":\"Event\",\"gipodId\":3773285}],\"description\":\"Eetwagen\",\"owner\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"contactOrganisation\":[],\"zone\":[{\"@id\":\"/api/v1/mobility-hindrances/10228787/zones/6dd8d314-9415-4103-bb06-c7c9fbf3dc19\",\"@type\":\"Zone\",\"consequence\":[{\"@id\":\"/api/v1/taxonomies/mobility-hindrance/consequencetypes/c53813ab-814f-4ff4-8a87-6934c72e175f\",\"prefLabel\":\"Geen doorgang voor gemotoriseerd verkeer\"}],\"geometry\":{\"@type\":\"Geometry\",\"wkt\":\" POLYGON ((109802.90525490482 189774.7184551563, 109801.10560758458 189779.91360264365, 109803.47330090578 189780.72935110983, 109805.27295029224 189775.5342052644, 109807.07260330272 189770.33905491605, 109804.70490584878 189769.52330316138, 109802.90525490482 189774.7184551563))\"},\"zoneType\":{\"@id\":\"/api/v1/taxonomies/zonetypes/0fb72ef7-6ac9-4a70-b295-a30ea215d250\",\"prefLabel\":\"HinderZone\"}}],\"period\":[{\"@type\":\"Period\",\"start\":\"2021-01-31T23:00:00Z\",\"end\":\"2021-02-03T22:30:00Z\"}],\"timeSchedule\":null,\"permittedBy\":[],\"status\":{\"@id\":\"/api/v1/taxonomies/statuses/a411c53e-db33-436a-9bb9-d62d535b661d\",\"prefLabel\":\"Onbekend\"},\"generatedAtTime\":\"2020-12-28T09:37:18.577Z\",\"eventName\":\"MobilityHindranceWasImportedFromLegacy\",\"isVersionOf\":\"/api/v1/mobility-hindrances/10228787\",\"memberOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"lastModifiedOn\":\"2020-12-07T15:10:22.18Z\",\"lastModifiedBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"createdOn\":\"2020-12-07T15:10:22.18Z\",\"createdBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"}},{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/3\",\"@type\":\"MobilityHindrance\",\"gipodId\":10228787,\"identifier\":[{\"@type\":\"Identifier\",\"Identifier.identifier\":{\"value\":\"10228787\",\"type\":\"gipodId\"},\"assignedByName\":\"https://gipod.vlaanderen.be\"}],\"isConsequenceOf\":[{\"@id\":\"/api/v1/events/3773285\",\"@type\":\"Event\",\"gipodId\":3773285}],\"description\":\"Eetwagen\",\"owner\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"contactOrganisation\":[],\"zone\":[{\"@id\":\"/api/v1/mobility-hindrances/10228787/zones/6dd8d314-9415-4103-bb06-c7c9fbf3dc19\",\"@type\":\"Zone\",\"consequence\":[{\"@id\":\"/api/v1/taxonomies/mobility-hindrance/consequencetypes/c53813ab-814f-4ff4-8a87-6934c72e175f\",\"prefLabel\":\"Geen doorgang voor gemotoriseerd verkeer\"}],\"geometry\":{\"@type\":\"Geometry\",\"wkt\":\" POLYGON ((109802.90525490482 189774.7184551563, 109801.10560758458 189779.91360264365, 109803.47330090578 189780.72935110983, 109805.27295029224 189775.5342052644, 109807.07260330272 189770.33905491605, 109804.70490584878 189769.52330316138, 109802.90525490482 189774.7184551563))\"},\"zoneType\":{\"@id\":\"/api/v1/taxonomies/zonetypes/0fb72ef7-6ac9-4a70-b295-a30ea215d250\",\"prefLabel\":\"HinderZone\"}}],\"period\":[{\"@type\":\"Period\",\"start\":\"2021-01-31T23:00:00Z\",\"end\":\"2021-02-03T22:30:00Z\"}],\"timeSchedule\":null,\"permittedBy\":[],\"status\":{\"@id\":\"/api/v1/taxonomies/statuses/a411c53e-db33-436a-9bb9-d62d535b661d\",\"prefLabel\":\"Onbekend\"},\"generatedAtTime\":\"2020-12-28T09:37:18.578Z\",\"eventName\":\"MobilityHindranceWasImportedFromLegacy\",\"isVersionOf\":\"/api/v1/mobility-hindrances/10228787\",\"memberOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"lastModifiedOn\":\"2020-12-07T15:10:22.18Z\",\"lastModifiedBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"},\"createdOn\":\"2020-12-07T15:10:22.18Z\",\"createdBy\":{\"isVersionOf\":\"/api/v1/organisations/0201f2a2-39e6-0a88-b5a3-c912f52a9a93\",\"@type\":\"Organisation\",\"preferredName\":\"Geosparc\"}}]}", "headers": { "Content-Type": "application/ld+json;charset=UTF-8", "Cache-Control": "public, max-age=1, mutable" diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-tree-node-not-ordered.json b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-tree-node-not-ordered.json new file mode 100644 index 000000000..930a77a3b --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/200-tree-node-not-ordered.json @@ -0,0 +1,19 @@ +{ + "request": { + "method": "GET", + "url": "/200-tree-node-not-ordered", + "headers": { + "Accept": { + "contains": "application/ld+json" + } + } + }, + "response": { + "status": 200, + "body": "{\"@context\":[\"https://private-api.gipod.beta-vlaanderen.be/api/v1/context/gipod.jsonld\"],\"@id\":\"http://localhost:10101/200-tree-node-not-ordered\",\"@type\":\"Node\",\"viewOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"collectionInfo\":{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"@type\":\"EventStream\",\"shape\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances/shape\",\"timestampPath\":\"prov:generatedAtTime\",\"versionOfPath\":\"dct:isVersionOf\"},\"tree:relation\":[{\"tree:node\":\"http://localhost:10101/200-second-tree-node\",\"@type\":\"tree:GreaterThanRelation\",\"tree:path\":\"prov:generatedAtTime\",\"tree:value\":\"2020-12-28T09:37:18.577Z\"}],\"items\":[{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/1\",\"@type\":\"MobilityHindrance\",\"gipodId\":10228530,\"identifier\":[{\"@type\":\"Identifier\",\"Identifier.identifier\":{\"value\":\"10228530\",\"type\":\"gipodId\"},\"assignedByName\":\"https://gipod.vlaanderen.be\"}],\"isConsequenceOf\":[{\"@id\":\"/api/v1/works/3779167\",\"@type\":\"Work\",\"gipodId\":3779167}],\"description\":\"8420 De Haan, Grotestraat 159: Stelling\",\"owner\":{\"isVersionOf\":\"/api/v1/organisations/fedab33f-792a-029c-9b34-9d9cfe7d6245\",\"@type\":\"Organisation\",\"preferredName\":\"EagleBe Smartcity\"},\"contactOrganisation\":[],\"zone\":[{\"@id\":\"/api/v1/mobility-hindrances/10228530/zones/991fa7c1-a530-44f7-a185-98ba45b69ce0\",\"@type\":\"Zone\",\"consequence\":[],\"geometry\":{\"@type\":\"Geometry\",\"wkt\":\" POLYGON ((57626.8718169853 218733.768758968, 57626.8718169853 218731.768758968, 57639.8718169853 218731.768758968, 57639.8718169853 218733.768758968, 57626.8718169853 218733.768758968))\"},\"zoneType\":{\"@id\":\"/api/v1/taxonomies/zonetypes/0fb72ef7-6ac9-4a70-b295-a30ea215d250\",\"prefLabel\":\"HinderZone\"}}],\"period\":[{\"@type\":\"Period\",\"start\":\"2021-01-14T05:00:00Z\",\"end\":\"2021-01-27T19:00:00Z\"}],\"timeSchedule\":null,\"permittedBy\":[],\"status\":{\"@id\":\"/api/v1/taxonomies/statuses/a411c53e-db33-436a-9bb9-d62d535b661d\",\"prefLabel\":\"Onbekend\"},\"generatedAtTime\":\"2020-12-28T10:36:09.72Z\",\"eventName\":\"MobilityHindranceWasImportedFromLegacy\",\"isVersionOf\":\"/api/v1/mobility-hindrances/10228530\",\"memberOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"lastModifiedOn\":\"2020-12-24T09:44:05.58Z\",\"lastModifiedBy\":{\"isVersionOf\":\"/api/v1/organisations/fedab33f-792a-029c-9b34-9d9cfe7d6245\",\"@type\":\"Organisation\",\"preferredName\":\"EagleBe Smartcity\"},\"createdOn\":\"2020-12-24T09:44:05.58Z\",\"createdBy\":{\"isVersionOf\":\"/api/v1/organisations/fedab33f-792a-029c-9b34-9d9cfe7d6245\",\"@type\":\"Organisation\",\"preferredName\":\"EagleBe Smartcity\"}},{\"@id\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/2\",\"@type\":\"MobilityHindrance\",\"gipodId\":10228530,\"identifier\":[{\"@type\":\"Identifier\",\"Identifier.identifier\":{\"value\":\"10228530\",\"type\":\"gipodId\"},\"assignedByName\":\"https://gipod.vlaanderen.be\"}],\"isConsequenceOf\":[{\"@id\":\"/api/v1/works/3779167\",\"@type\":\"Work\",\"gipodId\":3779167}],\"description\":\"8420 De Haan, Grotestraat 159: Stelling\",\"owner\":{\"isVersionOf\":\"/api/v1/organisations/fedab33f-792a-029c-9b34-9d9cfe7d6245\",\"@type\":\"Organisation\",\"preferredName\":\"EagleBe Smartcity\"},\"contactOrganisation\":[],\"zone\":[{\"@id\":\"/api/v1/mobility-hindrances/10228530/zones/991fa7c1-a530-44f7-a185-98ba45b69ce0\",\"@type\":\"Zone\",\"consequence\":[],\"geometry\":{\"@type\":\"Geometry\",\"wkt\":\" POLYGON ((57626.8718169853 218733.768758968, 57626.8718169853 218731.768758968, 57639.8718169853 218731.768758968, 57639.8718169853 218733.768758968, 57626.8718169853 218733.768758968))\"},\"zoneType\":{\"@id\":\"/api/v1/taxonomies/zonetypes/0fb72ef7-6ac9-4a70-b295-a30ea215d250\",\"prefLabel\":\"HinderZone\"}}],\"period\":[{\"@type\":\"Period\",\"start\":\"2021-01-14T05:00:00Z\",\"end\":\"2021-01-27T19:00:00Z\"}],\"timeSchedule\":null,\"permittedBy\":[],\"status\":{\"@id\":\"/api/v1/taxonomies/statuses/a411c53e-db33-436a-9bb9-d62d535b661d\",\"prefLabel\":\"Onbekend\"},\"generatedAtTime\":\"2020-12-28T09:36:09.72Z\",\"eventName\":\"MobilityHindranceWasImportedFromLegacy\",\"isVersionOf\":\"/api/v1/mobility-hindrances/10228530\",\"memberOf\":\"https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances\",\"lastModifiedOn\":\"2020-12-24T09:44:05.58Z\",\"lastModifiedBy\":{\"isVersionOf\":\"/api/v1/organisations/fedab33f-792a-029c-9b34-9d9cfe7d6245\",\"@type\":\"Organisation\",\"preferredName\":\"EagleBe Smartcity\"},\"createdOn\":\"2020-12-24T09:44:05.58Z\",\"createdBy\":{\"isVersionOf\":\"/api/v1/organisations/fedab33f-792a-029c-9b34-9d9cfe7d6245\",\"@type\":\"Organisation\",\"preferredName\":\"EagleBe Smartcity\"}}]}", + "headers": { + "Content-Type": "application/ld+json;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-1.json b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-1.json new file mode 100644 index 000000000..ce403f5b2 --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-1.json @@ -0,0 +1,18 @@ +{ + "request": { + "method": "GET", + "url": "/items/grouped?group=1" + }, + "response": { + "status": 200, + "bodyFileName": "items-group-1-fragment.ttl", + "transformers": [ + "response-template" + ], + "headers": { + "Content-Type": "text/turtle;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} + diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-2.json b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-2.json new file mode 100644 index 000000000..139555d0a --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-2.json @@ -0,0 +1,17 @@ +{ + "request": { + "method": "GET", + "url": "/items/grouped?group=2" + }, + "response": { + "status": 200, + "bodyFileName": "items-group-2-fragment.ttl", + "transformers": [ + "response-template" + ], + "headers": { + "Content-Type": "text/turtle;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-3.json b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-3.json new file mode 100644 index 000000000..29507e3d8 --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/resources/mappings/items-grouped-3.json @@ -0,0 +1,17 @@ +{ + "request": { + "method": "GET", + "url": "/items/grouped?group=3" + }, + "response": { + "status": 200, + "bodyFileName": "items-group-3-fragment.ttl", + "transformers": [ + "response-template" + ], + "headers": { + "Content-Type": "text/turtle;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} \ No newline at end of file diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index 63a0a5117..36b7e8b6e 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldi-common diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java index b7caefea3..458dd9022 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/JsonLdFrameWriter.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter; -import com.github.jsonldjava.core.JsonLdOptions; import org.apache.jena.atlas.json.JSON; import org.apache.jena.atlas.json.JsonObject; import org.apache.jena.rdf.model.Model; @@ -13,6 +12,8 @@ import java.io.OutputStream; +import com.github.jsonldjava.core.JsonLdOptions; + import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.PrefixAdder.addPrefixesToModel; public class JsonLdFrameWriter implements LdiRdfWriter { diff --git a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java similarity index 69% rename from ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampExtractor.java rename to ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java index 4c851a8b8..9547f27a6 100644 --- a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; +package be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor; import org.apache.jena.rdf.model.Model; diff --git a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromCurrentTimeExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java similarity index 77% rename from ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromCurrentTimeExtractor.java rename to ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java index 1b6500f0d..25c8c6c27 100644 --- a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromCurrentTimeExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; +package be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor; import org.apache.jena.rdf.model.Model; diff --git a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromPathExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java similarity index 94% rename from ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromPathExtractor.java rename to ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java index 22a1d300d..f637151db 100644 --- a/ldi-core/file-archiving/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromPathExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; +package be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor; import org.apache.jena.datatypes.RDFDatatype; import org.apache.jena.datatypes.xsd.XSDDateTime; diff --git a/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriterTest.java b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriterTest.java index bedff418d..87fa58f9e 100644 --- a/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriterTest.java +++ b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriterTest.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter; -import com.github.jsonldjava.core.JsonLdOptions; import org.apache.jena.atlas.json.JSON; import org.apache.jena.atlas.json.JsonObject; import org.apache.jena.rdf.model.Model; @@ -25,6 +24,8 @@ import java.util.Objects; import java.util.stream.Stream; +import com.github.jsonldjava.core.JsonLdOptions; + import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriterProperties.FRAME; import static org.assertj.core.api.Assertions.assertThat; diff --git a/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java index 5ed1dfdde..91aa0b344 100644 --- a/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java +++ b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/parser/JenaContextProviderTest.java @@ -37,4 +37,4 @@ void withCreate_ShouldContainDefaults() { assertThat(jsonLdOptions).isNotNull(); assertThat(jsonLdOptions.getDocumentCache()).isNotNull(); } -} \ No newline at end of file +} diff --git a/ldi-core/file-archiving/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromCurrentTimeExtractorTest.java b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractorTest.java similarity index 90% rename from ldi-core/file-archiving/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromCurrentTimeExtractorTest.java rename to ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractorTest.java index 7edafec00..38a17966b 100644 --- a/ldi-core/file-archiving/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromCurrentTimeExtractorTest.java +++ b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractorTest.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; +package be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; diff --git a/ldi-core/file-archiving/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromPathExtractorTest.java b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractorTest.java similarity index 94% rename from ldi-core/file-archiving/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromPathExtractorTest.java rename to ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractorTest.java index bf97ed0c8..766b5e4d2 100644 --- a/ldi-core/file-archiving/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/TimestampFromPathExtractorTest.java +++ b/ldi-core/ldi-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractorTest.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; +package be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Property; @@ -38,4 +38,4 @@ void ExtractTimestamp_ShouldTimestamp_WhenFound() { assertEquals(LocalDateTime.of(2022, 5, 20, 9, 58, 15), result); } -} \ No newline at end of file +} diff --git a/ldi-core/ldi-common/src/test/resources/model-with-timestamp20220520.nq b/ldi-core/ldi-common/src/test/resources/model-with-timestamp20220520.nq new file mode 100644 index 000000000..1fd51335b --- /dev/null +++ b/ldi-core/ldi-common/src/test/resources/model-with-timestamp20220520.nq @@ -0,0 +1,36 @@ + "In opmaak"@nl-be . +_:B9d973b9a5c2911f1ffbf70024806f47f " POLYGON ((170287.59 200940.75, 170287.57 200945.75, 170292.58 200945.77, 170292.6 200940.77, 170287.59 200940.75))"^^ . +_:B9d973b9a5c2911f1ffbf70024806f47f . + . + "Gemeente Berlaar" . + "Beperkte doorgang voor voetgangers"@nl-be . + . + . + _:B9d973b9a5c2911f1ffbf70024806f47f . + . + . +_:Ba6b523ef75c48b5351f80cf6470bd2f2 . +_:Ba6b523ef75c48b5351f80cf6470bd2f2 "10810464"^^ . +_:Ba6b523ef75c48b5351f80cf6470bd2f2 "https://gipod.vlaanderen.be"@nl-be . + "HinderZone"@nl-be . + . + . + "10810463"^^ . + . + "2022-05-20T09:58:15.8610896Z"^^ . + "2022-05-20T09:58:15.8646433Z"^^ . + . + . + . + "MobilityHindranceZoneWasAdded"@nl-be . + . + "omschrijving" . + . + "2022-05-20T09:58:15"^^ . + . + _:Ba6b523ef75c48b5351f80cf6470bd2f2 . + _:B292e5628e8711e26b8812c62967d6e9c . + "10810464"^^ . +_:B292e5628e8711e26b8812c62967d6e9c "2022-05-27T17:00:00Z"^^ . +_:B292e5628e8711e26b8812c62967d6e9c "2022-05-27T07:00:00Z"^^ . +_:B292e5628e8711e26b8812c62967d6e9c . \ No newline at end of file diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 45189801f..125214d69 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java b/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java index 5aa96cd19..6c7eb3669 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java +++ b/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valuobjects/LinkedDataModel.java @@ -4,20 +4,19 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.parser.JenaContextProvider; import be.vlaanderen.informatievlaanderen.ldes.ldi.valuobjects.properties.LinkedDataAttribute; import be.vlaanderen.informatievlaanderen.ldes.ldi.valuobjects.valueproperties.DateTimeValue; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParser; +import org.apache.jena.sparql.util.Context; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import com.fasterxml.jackson.annotation.*; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.jena.sparql.util.Context; - import static be.vlaanderen.informatievlaanderen.ldes.ldi.config.NgsiV2ToLdMapping.*; import static be.vlaanderen.informatievlaanderen.ldes.ldi.services.NgsiLdURIParser.toNgsiLdUri; diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index c16c64161..8b7f51981 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index 8eea879cc..2fbadcc48 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index 360200e7d..8c6c1c559 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index ea297fe14..bd4df20cf 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index 34d00776c..82aa2d7f9 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index 7d5abc484..adbd12eeb 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index 5b83addd5..a0300f571 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index 2175e111f..b72b7b103 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index 49a1fb069..18f54e445 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldes-discoverer diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index e6f91c155..818b44fe3 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index b754569e8..bfd692954 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index 88b98e749..32acf1987 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index 9c2f6a9f1..fad076b33 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,13 +6,18 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT archive-file-out nar + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-common + ${project.version} + be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi-common diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ArchiveFileOutProcessor.java b/ldi-nifi/ldi-nifi-processors/archive-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ArchiveFileOutProcessor.java index 4b88394b7..a35fdc6f6 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ArchiveFileOutProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ArchiveFileOutProcessor.java @@ -1,9 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldio.ArchiveFile; -import be.vlaanderen.informatievlaanderen.ldes.ldio.TimestampExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.TimestampFromCurrentTimeExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.TimestampFromPathExtractor; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFWriter; diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index 9df9201d3..330f69e8c 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index 87cea010f..d84a42074 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index 94e19ec96..6703c04d4 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/JsonToLdProcessorTest.java b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/JsonToLdProcessorTest.java index cadd87046..f5e15b290 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/JsonToLdProcessorTest.java +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/JsonToLdProcessorTest.java @@ -1,7 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.jupiter.api.Test; @@ -11,6 +10,8 @@ import java.nio.file.Path; import java.util.Map; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.FAILURE; @WireMockTest(httpPort = 10101) diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index b00f7f6b2..eec7341ed 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 @@ -17,6 +17,11 @@ ldi-nifi-common ${project.version} + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-common + ${project.version} + com.github.tomakehurst wiremock-jre8 diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index d8616961e..a5c9ba20b 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -9,6 +9,9 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorDecorator; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.LdesPropertiesExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; import io.github.resilience4j.retry.Retry; import ldes.client.treenodesupplier.MemberSupplier; import ldes.client.treenodesupplier.MemberSupplierImpl; @@ -66,7 +69,7 @@ public Set getRelationships() { @Override public final List getSupportedPropertyDescriptors() { - return List.of(DATA_SOURCE_URL, DATA_SOURCE_FORMAT, DATA_DESTINATION_FORMAT, KEEP_STATE, + return List.of(DATA_SOURCE_URLS, DATA_SOURCE_FORMAT, DATA_DESTINATION_FORMAT, KEEP_STATE, TIMESTAMP_PATH, STATE_PERSISTENCE_STRATEGY, STREAM_TIMESTAMP_PATH_PROPERTY, STREAM_VERSION_OF_PROPERTY, STREAM_SHAPE_PROPERTY, API_KEY_HEADER_PROPERTY, OAUTH_SCOPE, @@ -77,12 +80,15 @@ public final List getSupportedPropertyDescriptors() { @OnScheduled public void onScheduled(final ProcessContext context) { - String dataSourceUrl = LdesProcessorProperties.getDataSourceUrl(context); + List dataSourceUrls = LdesProcessorProperties.getDataSourceUrl(context); Lang dataSourceFormat = LdesProcessorProperties.getDataSourceFormat(context); final RequestExecutor requestExecutor = getRequestExecutorWithPossibleRetry(context); - LdesMetaData ldesMetaData = new LdesMetaData(dataSourceUrl, dataSourceFormat); + LdesMetaData ldesMetaData = new LdesMetaData(dataSourceUrls, dataSourceFormat); StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(context); - TreeNodeProcessor treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor); + String timestampPath = LdesProcessorProperties.getTimestampPath(context); + TimestampExtractor timestampExtractor = timestampPath.isBlank() ? new TimestampFromCurrentTimeExtractor() : + new TimestampFromPathExtractor(createProperty(timestampPath)); + TreeNodeProcessor treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor); boolean keepState = stateKept(context); if (useVersionMaterialisation(context)) { final var versionOfProperty = createProperty(getVersionOfProperty(context)); @@ -97,8 +103,8 @@ public void onScheduled(final ProcessContext context) { determineLdesProperties(ldesMetaData, requestExecutor, context); - LOGGER.info("LDES extraction processor {} with base url {} (expected LDES source format: {})", - context.getName(), dataSourceUrl, dataSourceFormat); + LOGGER.info("LDES Client processor {} configured to follow (sub)streams {} (expected LDES source format: {})", + context.getName(), dataSourceUrls, dataSourceFormat); } private RequestExecutor getRequestExecutorWithPossibleRetry(final ProcessContext context) { diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java index 43bc478b5..a4dc99d1a 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java @@ -6,9 +6,12 @@ import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.Validator; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,12 +34,12 @@ public final class LdesProcessorProperties { private LdesProcessorProperties() { } - public static final PropertyDescriptor DATA_SOURCE_URL = new PropertyDescriptor.Builder() - .name("DATA_SOURCE_URL") - .displayName("Data source url") - .description("Url to data source") + public static final PropertyDescriptor DATA_SOURCE_URLS = new PropertyDescriptor.Builder() + .name("DATA_SOURCE_URLS") + .displayName("Data source urls") + .description("Comma separated list of ldes endpoints. Must be part of same view of an LDES.") .required(true) - .addValidator(StandardValidators.URL_VALIDATOR) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) .build(); public static final PropertyDescriptor DATA_SOURCE_FORMAT = new PropertyDescriptor.Builder() @@ -97,6 +100,14 @@ private LdesProcessorProperties() { .defaultValue(FALSE.toString()) .build(); + public static final PropertyDescriptor TIMESTAMP_PATH = new PropertyDescriptor.Builder() + .name("TIMESTAMP_PATH") + .displayName("Property path determining the timestamp used to order the members within a fragment") + .required(false) + .addValidator(Validator.VALID) + .defaultValue("") + .build(); + public static final PropertyDescriptor STREAM_TIMESTAMP_PATH_PROPERTY = new PropertyDescriptor.Builder() .name("STREAM_TIMESTAMP_PATH_PROPERTY") .displayName("Stream TimestampPath property to FlowFile") @@ -226,8 +237,16 @@ private LdesProcessorProperties() { .addValidator(StandardValidators.URI_VALIDATOR) .build(); - public static String getDataSourceUrl(final ProcessContext context) { - return context.getProperty(DATA_SOURCE_URL).getValue(); + public static List getDataSourceUrl(final ProcessContext context) { + var urls = Arrays.stream(context.getProperty(DATA_SOURCE_URLS).getValue().split(",")) + .map(String::trim) + .toList(); + + if (urls.stream().allMatch(LdesProcessorProperties::isValidUrl)) { + return urls; + } else { + throw new IllegalArgumentException("Not a (valid list of) datasource url(s)"); + } } public static Lang getDataSourceFormat(final ProcessContext context) { @@ -238,6 +257,10 @@ public static Lang getDataDestinationFormat(final ProcessContext context) { return RDFLanguages.nameToLang(context.getProperty(DATA_DESTINATION_FORMAT).getValue()); } + public static String getTimestampPath(final ProcessContext context) { + return context.getProperty(TIMESTAMP_PATH).getValue(); + } + public static boolean streamTimestampPathProperty(final ProcessContext context) { return TRUE.equals(context.getProperty(STREAM_TIMESTAMP_PATH_PROPERTY).asBoolean()); } @@ -330,4 +353,13 @@ public static String getVersionOfProperty(final ProcessContext context) { return context.getProperty(VERSION_OF_PROPERTY).getValue(); } + private static boolean isValidUrl(String url) { + try { + new URI(url); + return true; + } catch (URISyntaxException e) { + return false; + } + } + } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java index f5a1bb420..ca684f1cb 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java @@ -62,7 +62,7 @@ void tearDown() { @ParameterizedTest @ArgumentsSource(MatchNumberOfFlowFilesArgumentsProvider.class) void shouldMatchNumberOfFlowFiles(String dataSourceUrl, int numberOfRuns) { - testRunner.setProperty("DATA_SOURCE_URL", dataSourceUrl); + testRunner.setProperty("DATA_SOURCE_URLS", dataSourceUrl); testRunner.setProperty("STATE_PERSISTENCE_STRATEGY", "SQLITE"); @@ -78,7 +78,7 @@ void shouldMatchNumberOfFlowFiles(String dataSourceUrl, int numberOfRuns) { @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) void when_NecessaryPropertiesAreSet_then_statePersistenceCanBeCreated(Map properties) { - testRunner.setProperty("DATA_SOURCE_URL", + testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z"); properties.forEach((key, value) -> testRunner.setProperty(key, value)); @@ -101,7 +101,7 @@ void when_NecessaryPropertiesAreSet_then_statePersistenceCanBeCreated(Map properties) { - testRunner.setProperty("DATA_SOURCE_URL", + testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z"); testRunner.setProperty("STATE_PERSISTENCE_STRATEGY", "MEMORY"); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); @@ -153,7 +153,7 @@ void shouldSupportDifferentHttpRequestExecutors(Map properties) @Test void shouldSupportRetry() { - testRunner.setProperty("DATA_SOURCE_URL", "http://localhost:10101/retry"); + testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/retry"); testRunner.setProperty("STATE_PERSISTENCE_STRATEGY", "MEMORY"); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); testRunner.setProperty("RETRIES_ENABLED", Boolean.TRUE.toString()); @@ -171,7 +171,7 @@ void shouldSupportRetry() { @Test void shouldSupportVersionMaterialisation() { - testRunner.setProperty("DATA_SOURCE_URL", + testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z"); testRunner.setProperty("STATE_PERSISTENCE_STRATEGY", "MEMORY"); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); @@ -196,7 +196,7 @@ void shouldSupportVersionMaterialisation() { @Test void when_runningLdesClientWithStreamPropertiesFlags_expectsLdesPropertiesInFlowFile() { - testRunner.setProperty("DATA_SOURCE_URL", + testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?scenario=gml-data"); testRunner.setProperty("STREAM_SHAPE_PROPERTY", Boolean.TRUE.toString()); diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java index 8fd044b61..affcfc251 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java @@ -8,8 +8,8 @@ import java.util.List; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.getDataSourceUrl; +import static org.junit.jupiter.api.Assertions.*; class LdesProcessorPropertiesTest { @@ -32,6 +32,16 @@ void test_getStatusesToRetry() { assertFalse(statusesToRetry.contains(500)); } + @Test + void test_getDatasourceUrl() { + assertDoesNotThrow(() -> getMockContext("http://localhost/endpoint")); + assertDoesNotThrow(() -> getMockContext("http://localhost/endpoint,http://localhost/other")); + var singleInvalidUri = getMockContext("inv alid"); + assertThrows(IllegalArgumentException.class, () -> getDataSourceUrl(singleInvalidUri)); + var multiInvalidUri = getMockContext("inv alid,http://localhost/other"); + assertThrows(IllegalArgumentException.class, () -> getDataSourceUrl(multiInvalidUri)); + } + private static MockProcessContext getMockContext(String value) { return new MockProcessContext() { @Override diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index 5834dfd7a..001aacc4f 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 69e656417..fd3689c38 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml index 6f6036bab..ead23f16e 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT rdf4j-repository-materialisation-processor diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index bbebb195b..31a703679 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index c9e876291..edac87e66 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index 285584ce4..3b6644652 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index 7c9abfdd1..02524b292 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index 24c30329d..7ede46f0e 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index bf5db8ade..7cfb42abd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java index 9e6a2baee..dd86934bb 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java @@ -14,7 +14,7 @@ import org.springframework.jms.config.SimpleJmsListenerEndpoint; public class LdioAmqpIn extends LdioInput implements MessageListener { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn"; + public static final String NAME = "Ldio:AmqpIn"; private static final Logger log = LoggerFactory.getLogger(LdioAmqpIn.class); private final String defaultContentType; diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index 366527291..67a1c525f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java index 88452a7b5..9df5ab34f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java @@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory; public class LdioArchiveFileIn extends LdioInput { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioArchiveFileIn"; + public static final String NAME = "Ldio:ArchiveFileIn"; private final Logger log = LoggerFactory.getLogger(LdioArchiveFileIn.class); private final ArchiveFileCrawler archiveFileCrawler; private final Lang sourceFormat; diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index ac7bd88df..89263f6ba 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java index 297570211..e4e956531 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiAzureBlobOut.java @@ -4,16 +4,16 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.conversionstrategy.ConversionStrategy; import be.vlaanderen.informatievlaanderen.ldes.ldio.conversionstrategy.ConversionStrategyFactory; import be.vlaanderen.informatievlaanderen.ldes.ldio.util.MemberIdExtractor; -import org.apache.jena.rdf.model.Model; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.azure.core.util.BinaryData; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.specialized.BlockBlobClient; +import org.apache.jena.rdf.model.Model; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SuppressWarnings("java:S2629") public class LdiAzureBlobOut implements LdiOutput { + public static final String NAME = "Ldio:AzureBlobOut"; private final Logger log = LoggerFactory.getLogger(LdiAzureBlobOut.class); private final String outputLanguage; diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAzureBlobOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAzureBlobOutAutoConfig.java index 731b78616..684442879 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAzureBlobOutAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAzureBlobOutAutoConfig.java @@ -7,10 +7,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdiAzureBlobOut.NAME; + @Configuration public class LdioAzureBlobOutAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdiAzureBlobOut") + @Bean(NAME) public LdioOutputConfigurator ldioConfigurator() { return new LdioAzureBlobOutConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 1fa4800be..79868ed00 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiConsoleOut.java b/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiConsoleOut.java index 4a7c2e2ee..cb12d68c3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiConsoleOut.java +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdiConsoleOut.java @@ -10,6 +10,7 @@ @SuppressWarnings("java:S2629") public class LdiConsoleOut implements LdiOutput { + public static final String NAME = "Ldio:ConsoleOut"; private final Logger log = LoggerFactory.getLogger(LdiConsoleOut.class); private final LdiRdfWriterProperties properties; diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConsoleOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConsoleOutAutoConfig.java index 9abfe6ec9..91327b72d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConsoleOutAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConsoleOutAutoConfig.java @@ -9,11 +9,12 @@ import org.springframework.context.annotation.Configuration; import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriterProperties.RDF_WRITER; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdiConsoleOut.NAME; @Configuration public class LdioConsoleOutAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioConsoleOut") + @Bean(NAME) public LdioOutputConfigurator ldioConfigurator() { return new LdioConsoleOutConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index 0998a191e..9678ab242 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioFileOut.java b/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioFileOut.java index 2a1889309..4077309a3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioFileOut.java +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioFileOut.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriter; import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriterProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; @@ -12,7 +13,7 @@ @SuppressWarnings("java:S112") public class LdioFileOut implements LdiOutput { - + public static final String NAME = "Ldio:FileOut"; private final TimestampExtractor timestampExtractor; private final String archiveRootDir; diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioFileOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioFileOutAutoConfig.java index b75816b10..b06fc71b7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioFileOutAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioFileOutAutoConfig.java @@ -1,10 +1,10 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioFileOut; -import be.vlaanderen.informatievlaanderen.ldes.ldio.TimestampExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.TimestampFromCurrentTimeExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.TimestampFromPathExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioOutputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.apache.jena.rdf.model.ResourceFactory; @@ -13,6 +13,8 @@ import java.io.File; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioFileOut.NAME; + @Configuration public class LdioFileOutAutoConfig { @@ -20,7 +22,7 @@ public class LdioFileOutAutoConfig { public static final String TIMESTAMP_PATH_PROP = "timestamp-path"; @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioFileOut") + @Bean(NAME) public LdioOutputConfigurator ldiFileOutConfigurator() { return new LdioOutputConfigurator() { @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index 12fa50a3e..38658357f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWkt.java b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioGeoJsonToWkt.java similarity index 79% rename from ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWkt.java rename to ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioGeoJsonToWkt.java index 70bb686d2..c56469887 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWkt.java +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioGeoJsonToWkt.java @@ -1,10 +1,11 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; import org.apache.jena.rdf.model.Model; public class LdioGeoJsonToWkt extends LdioTransformer { + public static final String NAME = "Ldio:GeoJsonToWktTransformer"; private final GeoJsonToWktTransformer transformer; public LdioGeoJsonToWkt() { diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWktAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWktAutoConfig.java index c19266fef..7d69beed8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWktAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioGeoJsonToWktAutoConfig.java @@ -1,13 +1,16 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioGeoJsonToWkt; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioGeoJsonToWkt.NAME; + @Configuration public class LdioGeoJsonToWktAutoConfig { - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer") + @Bean(NAME) public LdioTransformerConfigurator geoJsonToWktConfigurator() { return properties -> new LdioGeoJsonToWkt(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index f26919e44..af7331791 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpEnricher.java b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpEnricher.java index 16f1537d1..605b2b0ab 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpEnricher.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpEnricher.java @@ -15,7 +15,7 @@ import java.util.List; public class LdioHttpEnricher extends LdioTransformer { - + public static final String NAME = "Ldio:HttpEnricher"; private static final Logger log = LoggerFactory.getLogger(LdioHttpEnricher.class); private final LdiAdapter adapter; diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpEnricherAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpEnricherAutoConfig.java index d12d547b4..cc16eb6d5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpEnricherAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpEnricherAutoConfig.java @@ -12,13 +12,14 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpEnricherProperties.ADAPTER_CONFIG; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpEnricherProperties.ADAPTER_NAME; @Configuration public class LdioHttpEnricherAutoConfig { - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher") + @Bean(NAME) public LdioTransformerConfigurator ldioConfigurator(ConfigurableApplicationContext configContext) { return config -> { final LdiAdapter adapter = createAdapter(configContext, config); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index db560261b..3540fc71f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java similarity index 87% rename from ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java rename to ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java index cf84b153c..4b00df84b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java @@ -22,17 +22,17 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval.TYPE.CRON; -public class HttpInputPoller extends LdioInput implements Runnable { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller"; +public class LdioHttpInputPoller extends LdioInput implements Runnable { + public static final String NAME = "Ldio:HttpInPoller"; private final ThreadPoolTaskScheduler scheduler; private final RequestExecutor requestExecutor; private final List requests; private final boolean continueOnFail; - private static final Logger log = LoggerFactory.getLogger(HttpInputPoller.class); + private static final Logger log = LoggerFactory.getLogger(LdioHttpInputPoller.class); private static final String CONTENT_TYPE = "Content-Type"; - public HttpInputPoller(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, List endpoints, - boolean continueOnFail, RequestExecutor requestExecutor) { + public LdioHttpInputPoller(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, List endpoints, + boolean continueOnFail, RequestExecutor requestExecutor) { super(NAME, pipelineName, executor, adapter, observationRegistry); this.requestExecutor = requestExecutor; this.requests = endpoints.stream().map(endpoint -> new GetRequest(endpoint, RequestHeaders.empty())).toList(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java similarity index 79% rename from ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java rename to ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java index 7131fcc6f..92450d3c1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldio.HttpInputPoller; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInputPoller; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; @@ -13,15 +13,16 @@ import java.util.List; import java.util.Optional; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.HttpInputPollerProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInputPoller.NAME; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties.*; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; @Configuration -public class HttpInputPollerAutoConfig { +public class LdioHttpInputPollerAutoConfig { private static final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier(); @SuppressWarnings("java:S6830") - @Bean(HttpInputPoller.NAME) + @Bean(NAME) public HttpInputPollerConfigurator httpInputPollerConfigurator(ObservationRegistry observationRegistry) { return new HttpInputPollerConfigurator(observationRegistry); } @@ -34,8 +35,8 @@ public HttpInputPollerConfigurator(ObservationRegistry observationRegistry) { } @Override - public HttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, - ComponentProperties properties) { + public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, + ComponentProperties properties) { String pipelineName = properties.getProperty(PIPELINE_NAME); List endpoints = properties.getPropertyList(URL); @@ -43,7 +44,7 @@ public HttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); - var httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, observationRegistry, endpoints, continueOnFail, requestExecutor); + var httpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, observationRegistry, endpoints, continueOnFail, requestExecutor); httpInputPoller.schedulePoller(getPollingInterval(properties)); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerProperties.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java similarity index 85% rename from ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerProperties.java rename to ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java index 57e73624d..00833bba4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerProperties.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java @@ -1,8 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; -public class HttpInputPollerProperties { +public class LdioHttpInputPollerProperties { - private HttpInputPollerProperties() { + private LdioHttpInputPollerProperties() { } public static final String URL = "url"; diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/PollingInterval.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/PollingInterval.java index 68b003151..fe637e6f8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/PollingInterval.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/PollingInterval.java @@ -9,7 +9,7 @@ import java.time.format.DateTimeParseException; import java.util.Objects; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.HttpInputPollerProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties.*; public class PollingInterval { private static final Logger logger = LoggerFactory.getLogger(PollingInterval.class); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java similarity index 83% rename from ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java rename to ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java index 3ca40cdf6..2c211e1f4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java @@ -35,7 +35,7 @@ import static org.mockito.Mockito.*; @WireMockTest(httpPort = 10101) -class HttpInputPollerTest { +class LdioHttpInputPollerTest { private final LdiAdapter adapter = mock(LdiAdapter.class); private final ComponentExecutor executor = mock(ComponentExecutor.class); @@ -44,7 +44,7 @@ class HttpInputPollerTest { private static final String ENDPOINT = "/resource"; private static final String CONTENT = "_:b0 \"Jane Doe\" ."; private static final String CONTENT_TYPE = "application/n-quads"; - private HttpInputPoller httpInputPoller; + private LdioHttpInputPoller ldioHttpInputPoller; private static final RequestExecutor noAuthExecutor = new RequestExecutorFactory().createNoAuthExecutor(); @BeforeEach @@ -57,12 +57,12 @@ void setUp() { .thenReturn(Stream.of()) .thenReturn(Stream.of()); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor); + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor); } @AfterEach void tearDown() { - httpInputPoller.shutdown(); + ldioHttpInputPoller.shutdown(); } @Test @@ -70,7 +70,7 @@ void testClientPolling() { stubFor(get(ENDPOINT).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - httpInputPoller.run(); + ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(ENDPOINT))); } @@ -78,8 +78,8 @@ void testClientPolling() { void whenPolling_andMissesHeader() { stubFor(get(ENDPOINT).willReturn(ok().withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); - Executable polling = () -> httpInputPoller.run(); + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); + Executable polling = () -> ldioHttpInputPoller.run(); assertThrows(MissingHeaderException.class, polling); Mockito.verifyNoInteractions(adapter); @@ -91,7 +91,7 @@ void whenPolling_andMissesHeader() { void whenPeriodicPolling_thenReturnTwoTimesTheSameResponse(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - httpInputPoller.schedulePoller(pollingInterval); + ldioHttpInputPoller.schedulePoller(pollingInterval); Mockito.verify(adapter, timeout(4000).times(2)).apply(LdiAdapter.Content.of(CONTENT, CONTENT_TYPE)); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), getRequestedFor(urlEqualTo(ENDPOINT))); @@ -102,10 +102,10 @@ void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldSti stubFor(get(ENDPOINT).willReturn(serverError().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); String otherEndpoint = "/other-resource"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), true, noAuthExecutor); - httpInputPoller.run(); + ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(ENDPOINT))); WireMock.verify(getRequestedFor(urlEqualTo(otherEndpoint))); @@ -118,10 +118,10 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse(Poll stubFor(get(endpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); String otherEndpoint = "/other-endpoint"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), true, noAuthExecutor); - httpInputPoller.schedulePoller(pollingInterval); + ldioHttpInputPoller.schedulePoller(pollingInterval); Mockito.verify(adapter, timeout(6000).times(4)).apply(LdiAdapter.Content.of(CONTENT, CONTENT_TYPE)); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), getRequestedFor(urlEqualTo(endpoint))); @@ -133,7 +133,7 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse(Poll void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(forbidden())); - httpInputPoller.schedulePoller(pollingInterval); + ldioHttpInputPoller.schedulePoller(pollingInterval); Mockito.verify(adapter, after(4000).never()).apply(any()); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), @@ -146,8 +146,8 @@ void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling(PollingI void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(forbidden())); - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); - httpInputPoller.schedulePoller(pollingInterval); + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); + ldioHttpInputPoller.schedulePoller(pollingInterval); Mockito.verify(adapter, after(4000).never()).apply(any()); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 1), getRequestedFor(urlEqualTo(ENDPOINT))); @@ -156,10 +156,10 @@ void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling(Polling @Test void when_EndpointDoesNotExist_Then_NoDataIsSent() { String wrongEndpoint = "/non-existing-resource"; - httpInputPoller = new HttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + wrongEndpoint), true, + ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + wrongEndpoint), true, noAuthExecutor); - httpInputPoller.run(); + ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(wrongEndpoint))); Mockito.verifyNoInteractions(adapter); @@ -169,7 +169,7 @@ void when_EndpointDoesNotExist_Then_NoDataIsSent() { void when_ResponseIsNot200_Then_NoDataIsSent() { stubFor(get(ENDPOINT).willReturn(forbidden())); - httpInputPoller.run(); + ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(ENDPOINT))); Mockito.verifyNoInteractions(adapter); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java similarity index 75% rename from ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java rename to ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java index 9f3cbe0d0..92198f8f8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java @@ -4,8 +4,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.PollingIntervalTest.InvalidCronArgumentsProvider; import be.vlaanderen.informatievlaanderen.ldes.ldio.PollingIntervalTest.InvalidIntervalArgumentsProvider; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.HttpInputPollerAutoConfig; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.HttpInputPollerProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerAutoConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.junit.jupiter.api.Test; @@ -23,7 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.*; -class HttpInputPollerAutoConfigTest { +class LdioLdioHttpInputPollerAutoConfigTest { private final LdiAdapter adapter = mock(LdiAdapter.class); private final ComponentExecutor executor = mock(ComponentExecutor.class); @@ -31,15 +31,15 @@ class HttpInputPollerAutoConfigTest { private static final String ENDPOINT = "/resource"; private static ComponentProperties createConfigWithInterval(String url, String interval, String continueOnFail) { - return new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", HttpInputPollerProperties.URL, url, - HttpInputPollerProperties.INTERVAL, interval, - HttpInputPollerProperties.CONTINUE_ON_FAIL, continueOnFail)); + return new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", LdioHttpInputPollerProperties.URL, url, + LdioHttpInputPollerProperties.INTERVAL, interval, + LdioHttpInputPollerProperties.CONTINUE_ON_FAIL, continueOnFail)); } private static ComponentProperties createConfigWithCron(String url, String cron, String continueOnFail) { - return new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", HttpInputPollerProperties.URL, url, - HttpInputPollerProperties.CRON, cron, - HttpInputPollerProperties.CONTINUE_ON_FAIL, continueOnFail)); + return new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", LdioHttpInputPollerProperties.URL, url, + LdioHttpInputPollerProperties.CRON, cron, + LdioHttpInputPollerProperties.CONTINUE_ON_FAIL, continueOnFail)); } private static ComponentProperties createDefaultISOTestConfig() { @@ -52,8 +52,8 @@ private static ComponentProperties createDefaultCronTestConfig() { @Test void when_ValidIntervalConfig() { - try (MockedConstruction ignored = mockConstruction(HttpInputPoller.class)) { - HttpInputPoller poller = new HttpInputPollerAutoConfig() + try (MockedConstruction ignored = mockConstruction(LdioHttpInputPoller.class)) { + LdioHttpInputPoller poller = new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) .configure(adapter, executor, createDefaultISOTestConfig()); verify(poller, times(1)).schedulePoller(new PollingInterval(Duration.of(1, ChronoUnit.SECONDS))); @@ -62,8 +62,8 @@ void when_ValidIntervalConfig() { @Test void when_ValidCronConfig() { - try (MockedConstruction ignored = mockConstruction(HttpInputPoller.class)) { - HttpInputPoller poller = new HttpInputPollerAutoConfig() + try (MockedConstruction ignored = mockConstruction(LdioHttpInputPoller.class)) { + LdioHttpInputPoller poller = new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) .configure(adapter, executor, createDefaultCronTestConfig()); verify(poller, times(1)).schedulePoller(new PollingInterval(new CronTrigger("* * * * * *"))); @@ -73,7 +73,7 @@ void when_ValidCronConfig() { @ParameterizedTest @ArgumentsSource(InvalidIntervalArgumentsProvider.class) void whenInvalidIntervalConfigured_thenCatchException(String interval) { - Executable configurePoller = () -> new HttpInputPollerAutoConfig() + Executable configurePoller = () -> new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) .configure(adapter, executor, createConfigWithInterval(BASE_URL + ENDPOINT, interval, "false")); @@ -83,7 +83,7 @@ void whenInvalidIntervalConfigured_thenCatchException(String interval) { @ParameterizedTest @ArgumentsSource(InvalidCronArgumentsProvider.class) void whenInvalidCronConfigured_thenCatchException(String cron) { - Executable configurePoller = () -> new HttpInputPollerAutoConfig() + Executable configurePoller = () -> new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) .configure(adapter, executor, createConfigWithCron(BASE_URL + ENDPOINT, cron, "false")); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 33c648489..dd15c30cd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java index 7ecb68790..0060665eb 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java @@ -6,7 +6,7 @@ import io.micrometer.observation.ObservationRegistry; public class LdioHttpInProcess extends LdioInput { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpIn"; + public static final String NAME = "Ldio:HttpIn"; public LdioHttpInProcess(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry) { super(NAME, pipelineName, executor, adapter, observationRegistry); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java index 87b0b176a..3595952f9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java @@ -11,6 +11,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInProcess.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.exception.LdiAdapterMissingException.verifyAdapterPresent; @@ -18,7 +19,7 @@ public class LdioHttpInAutoConfig { @SuppressWarnings("java:S6830") - @Bean(LdioHttpInProcess.NAME) + @Bean(NAME) public LdioHttpInConfigurator ldioConfigurator(ApplicationEventPublisher eventPublisher, ObservationRegistry observationRegistry) { return new LdioHttpInConfigurator(eventPublisher, observationRegistry); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index 6e01b7f01..6025bb3b9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpOut.java b/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpOut.java index 6a7dd8893..6f156e525 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpOut.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpOut.java @@ -17,7 +17,7 @@ import java.util.List; public class LdioHttpOut implements LdiOutput { - + public static final String NAME = "Ldio:HttpOut"; private static final Logger log = LoggerFactory.getLogger(LdioHttpOut.class); private final RequestExecutor requestExecutor; diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpOutAutoConfig.java index e24218aed..6dd8ed934 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpOutAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpOutAutoConfig.java @@ -11,12 +11,13 @@ import org.springframework.context.annotation.Configuration; import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriterProperties.RDF_WRITER; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut.NAME; @Configuration public class LdioHttpOutAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut") + @Bean(NAME) public LdioOutputConfigurator ldiHttpOutConfigurator() { return new LdioHttpOutConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index 7c91326dd..dd503a165 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioJsonToLdAdapterAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioJsonToLdAdapterAutoConfig.java index 2d006aecd..020f9ef68 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioJsonToLdAdapterAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioJsonToLdAdapterAutoConfig.java @@ -12,7 +12,7 @@ public class LdioJsonToLdAdapterAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter") + @Bean("Ldio:JsonToLdAdapter") public LdioAdapterConfigurator ldioJsonToLdAdapterConfigurator() { return new LdioJsonToLdConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 69c3f8b1a..4336aa0c8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java index af3f92341..91678fca3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java @@ -12,7 +12,7 @@ import org.springframework.kafka.listener.MessageListener; public class LdioKafkaIn extends LdioInput implements MessageListener { - + public static final String NAME = "Ldio:KafkaIn"; private static final Logger log = LoggerFactory.getLogger(LdioKafkaIn.class); private final String defaultContentType; diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaOut.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaOut.java index 2a6759652..27f519ab9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaOut.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaOut.java @@ -13,7 +13,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.KafkaOutConfigKeys.CONTENT_TYPE; public class LdioKafkaOut implements LdiOutput { - + public static final String NAME = "Ldio:KafkaOut"; private final KafkaTemplate kafkaTemplate; private final Lang lang; private final String topic; diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java index e794a4da5..74a0431a9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn.NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.KafkaInConfigKeys.*; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig.ORCHESTRATOR_NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; @@ -29,7 +30,6 @@ @Configuration public class LdioKafkaInAutoConfig { - private static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn"; @SuppressWarnings("java:S6830") @Bean(NAME) diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaOutAutoConfig.java index ca246d126..2e121ef31 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaOutAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaOutAutoConfig.java @@ -4,11 +4,13 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaOut.NAME; + @Configuration public class LdioKafkaOutAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaOut") + @Bean(NAME) public LdioOutputConfigurator ldiKafkaOutConfigurator() { return new LdioKafkaOutProcessorConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index 47398d544..ae71d5536 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 256f4bcad..a0d428356 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -24,7 +24,7 @@ @Configuration public class LdioLdesClientConnectorAutoConfig { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnector"; + public static final String NAME = "Ldio:LdesClientConnector"; @Bean(NAME) public LdioInputConfigurator ldioConfigurator(ApplicationEventPublisher eventPublisher, diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 0410883dd..c98424b17 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 045ff6962..699504ad3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -14,7 +14,7 @@ public class LdioLdesClient extends LdioInput { - public static final String NAME = "be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient"; + public static final String NAME = "Ldio:LdesClient"; private final Logger log = LoggerFactory.getLogger(LdioLdesClient.class); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java index 390096411..a46426cd0 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java @@ -6,7 +6,7 @@ private LdioLdesClientProperties() { } // general properties - public static final String URL = "url"; + public static final String URLS = "urls"; public static final String SOURCE_FORMAT = "source-format"; public static final String STATE = "state"; @@ -14,6 +14,7 @@ private LdioLdesClientProperties() { public static final String POSTGRES_USERNAME = "postgres.username"; public static final String POSTGRES_PASSWORD = "postgres.password"; public static final String POSTGRES_URL = "postgres.url"; + public static final String TIMESTAMP_PATH_PROP = "timestamp-path"; // version materialisation properties public static final String USE_VERSION_MATERIALISATION = "use-version-materialisation"; diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java index c68e76a4b..3491264d4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java @@ -2,7 +2,9 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import ldes.client.treenodesupplier.MemberSupplier; import ldes.client.treenodesupplier.MemberSupplierImpl; @@ -17,7 +19,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.*; +import static org.apache.jena.rdf.model.ResourceFactory.createProperty; public class MemberSupplierFactory { @@ -44,10 +49,15 @@ public MemberSupplier getMemberSupplier() { } private TreeNodeProcessor getTreeNodeProcessor() { - String targetUrl = properties.getProperty(LdioLdesClientProperties.URL); + List targetUrls = properties.getPropertyList(URLS); + Lang sourceFormat = getSourceFormat(); - LdesMetaData ldesMetaData = new LdesMetaData(targetUrl, sourceFormat); - return new TreeNodeProcessor(ldesMetaData, getStatePersistence(), requestExecutor); + LdesMetaData ldesMetaData = new LdesMetaData(targetUrls, sourceFormat); + TimestampExtractor timestampExtractor = properties.getOptionalProperty(TIMESTAMP_PATH_PROP) + .map(timestampPath -> (TimestampExtractor) new TimestampFromPathExtractor(createProperty(timestampPath))) + .orElseGet(TimestampFromCurrentTimeExtractor::new); + + return new TreeNodeProcessor(ldesMetaData, getStatePersistence(), requestExecutor, timestampExtractor); } private StatePersistence getStatePersistence() { @@ -55,7 +65,7 @@ private StatePersistence getStatePersistence() { } private Boolean getKeepState() { - return properties.getOptionalBoolean(LdioLdesClientProperties.KEEP_STATE).orElse(false); + return properties.getOptionalBoolean(KEEP_STATE).orElse(false); } private boolean useVersionMaterialisation() { @@ -70,7 +80,7 @@ private boolean useVersionMaterialisation() { } private Lang getSourceFormat() { - return properties.getOptionalProperty(LdioLdesClientProperties.SOURCE_FORMAT) + return properties.getOptionalProperty(SOURCE_FORMAT) .map(RDFLanguages::nameToLang) .orElse(Lang.JSONLD); } @@ -79,7 +89,7 @@ private VersionMaterialiser createVersionMaterialiser() { final Property versionOfProperty = properties .getOptionalProperty(VERSION_OF_PROPERTY) .map(ResourceFactory::createProperty) - .orElseGet(() -> ResourceFactory.createProperty("http://purl.org/dc/terms/isVersionOf")); + .orElseGet(() -> createProperty("http://purl.org/dc/terms/isVersionOf")); final boolean restrictToMembers = properties.getOptionalBoolean(RESTRICT_TO_MEMBERS).orElse(false); return new VersionMaterialiser(versionOfProperty, restrictToMembers); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java index a69df6b6e..e3d588dce 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java @@ -3,18 +3,22 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import com.github.tomakehurst.wiremock.WireMockServer; import io.cucumber.java.BeforeAll; +import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; import org.apache.jena.rdf.model.Model; import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; -import com.github.tomakehurst.wiremock.WireMockServer; - +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.URLS; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static org.awaitility.Awaitility.await; @@ -22,18 +26,36 @@ public class LdioLdesClientITSteps { private List members; static WireMockServer wireMockServer = new WireMockServer(options().port(10101)); + private final Map componentPropsMap = new HashMap<>(); @BeforeAll public static void before_all() { wireMockServer.start(); } - @Given("I start an ldes-ldio-in component with url {string}") - public void iStartAnLdesLdioInComponentWithUrl(String url) { + @Given("I want to follow the following LDES") + public void iWantToFollowTheFollowingLDES(List urls) { + AtomicInteger counter = new AtomicInteger(); + + urls.forEach(url -> { + String key = "%s.%d".formatted(URLS, counter.getAndIncrement()); + componentPropsMap.put(key, wireMockServer.baseUrl() + url); + }); + } + + @And("I configure this to be of RDF format {string}") + public void iConfigureThisToBeOfRDFFormat(String contentType) { + componentPropsMap.put(LdioLdesClientProperties.SOURCE_FORMAT, contentType); + } + + @When("^I start an ldes-ldio-in component") + public void iStartAnLdesLdioInComponentWithUrl() { members = new ArrayList<>(); ComponentExecutor componentExecutor = linkedDataModel -> members.add(linkedDataModel); - var props = new ComponentProperties(Map.of(PIPELINE_NAME, "pipeline", - LdioLdesClientProperties.URL, wireMockServer.baseUrl() + url)); + + componentPropsMap.put(PIPELINE_NAME, "pipeline"); + + var props = new ComponentProperties(componentPropsMap); var ldioInputConfigurator = new LdioLdesClientAutoConfig().ldioConfigurator(null); ldioInputConfigurator.configure(null, componentExecutor, props); } @@ -45,4 +67,5 @@ public void allMembersFromTheStreamArePassedToThePipeline(int memberCount) { return members.size() == memberCount; }); } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-group-1-fragment.ttl b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-group-1-fragment.ttl new file mode 100644 index 000000000..ad1264507 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-group-1-fragment.ttl @@ -0,0 +1,36 @@ +@prefix tree: . +@prefix rdf: . +@prefix ldes: . +@prefix schema: . +@prefix terms: . +@prefix prov: . + + + rdf:type tree:Node ; + terms:isPartOf . + + + + rdf:type ldes:EventStream ; + tree:member , , , + . + + + a schema:Person ; + schema:name "Jane Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.000Z"^^ . + + + a schema:Person ; + schema:name "Jack Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.001Z"^^ . + + + a schema:Person ; + schema:name "Joe Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.002Z"^^ . + + + a schema:Person ; + schema:name "James Deer" ; + prov:generatedAtTime "2024-02-07T00:00:00.003Z"^^ . \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-group-2-fragment.ttl b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-group-2-fragment.ttl new file mode 100644 index 000000000..e0d69588c --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-group-2-fragment.ttl @@ -0,0 +1,36 @@ +@prefix tree: . +@prefix rdf: . +@prefix ldes: . +@prefix schema: . +@prefix terms: . +@prefix prov: . + + + rdf:type tree:Node ; + terms:isPartOf . + + + + rdf:type ldes:EventStream ; + tree:member , , , + . + + + a schema:Person ; + schema:name "Jane Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.000Z"^^ . + + + a schema:Person ; + schema:name "Jack Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.001Z"^^ . + + + a schema:Person ; + schema:name "Joe Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.002Z"^^ . + + + a schema:Person ; + schema:name "James Doe" ; + prov:generatedAtTime "2024-02-07T00:00:00.003Z"^^ . diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature index 69480e0a0..92bf5bcd1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature @@ -3,5 +3,15 @@ Feature: LdesClientIntegrationTest I want to be able to use an LDES Client as input for LDIO Scenario: Reading an LDES Stream - Given I start an ldes-ldio-in component with url "/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z" - Then All 6 members from the stream are passed to the pipeline \ No newline at end of file + Given I want to follow the following LDES + | /exampleData?generatedAtTime=2022-05-03T00:00:00.000Z | + When I start an ldes-ldio-in component + Then All 6 members from the stream are passed to the pipeline + + Scenario: Reading multiple LDES Streams + Given I want to follow the following LDES + | /items/grouped?group=1 | + | /items/grouped?group=2 | + And I configure this to be of RDF format "text/turtle" + When I start an ldes-ldio-in component + Then All 8 members from the stream are passed to the pipeline \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items-grouped-1.json b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items-grouped-1.json new file mode 100644 index 000000000..2cf7271b3 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items-grouped-1.json @@ -0,0 +1,18 @@ +{ + "request": { + "method": "GET", + "url": "/items/grouped?group=1" + }, + "response": { + "status": 200, + "bodyFileName": "items-group-1-fragment.ttl", + "transformers": [ + "response-template" + ], + "headers": { + "Content-Type": "application/ld+json;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} + diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items-grouped-2.json b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items-grouped-2.json new file mode 100644 index 000000000..f1085a942 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items-grouped-2.json @@ -0,0 +1,17 @@ +{ + "request": { + "method": "GET", + "url": "/items/grouped?group=2" + }, + "response": { + "status": 200, + "bodyFileName": "items-group-2-fragment.ttl", + "transformers": [ + "response-template" + ], + "headers": { + "Content-Type": "application/ld+json;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index e8b83d214..e5ab20638 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter @@ -15,7 +15,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ngsiv2-to-ld-adapter - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioNgsiv2ToLdAdapterAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioNgsiv2ToLdAdapterAutoConfig.java index 4e2bee041..527a47dc3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioNgsiv2ToLdAdapterAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioNgsiv2ToLdAdapterAutoConfig.java @@ -11,8 +11,8 @@ public class LdioNgsiv2ToLdAdapterAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.NgsiV2ToLdAdapter") - public LdioAdapterConfigurator ldiHttpOutConfigurator() { + @Bean("Ldio:NgsiV2ToLdAdapter") + public LdioAdapterConfigurator ldioAdapterConfigurator() { return new LdioSparqlConstructProcessorConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index be0df49e3..269d6a7f5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRdfAdapterAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRdfAdapterAutoConfig.java index 39f6af642..45f289dcd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRdfAdapterAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRdfAdapterAutoConfig.java @@ -14,8 +14,8 @@ public class LdioRdfAdapterAutoConfig { public static final String MAX_JSONLD_CACHE_CAPACITY = "max-jsonld-cache-capacity"; @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter") - public LdioAdapterConfigurator ldiHttpOutConfigurator() { + @Bean("Ldio:RdfAdapter") + public LdioAdapterConfigurator ldioAdapterConfigurator() { return new LdioRdfConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml index 3d3a86446..a38a53b47 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-repository-materialiser diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java index 8ec074336..dda2fe63a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java @@ -8,7 +8,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioRepositoryMaterialiserProperties.*; public class LdioRepositoryMaterialiser implements LdiOutput { - + public static final String NAME = "Ldio:RepositoryMaterialiser"; private final Materialiser materialiser; public LdioRepositoryMaterialiser(ComponentProperties config) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LioRepositoryMaterialiserAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LioRepositoryMaterialiserAutoConfig.java index 6e9b14df3..f256a4f05 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LioRepositoryMaterialiserAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LioRepositoryMaterialiserAutoConfig.java @@ -7,11 +7,13 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioRepositoryMaterialiser.NAME; + @Configuration public class LioRepositoryMaterialiserAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser") + @Bean(NAME) public LdioOutputConfigurator ldiRepoMaterialiserConfigurator() { return new LdiRepoMaterialiserProcessorConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index 065873b1c..d543924bc 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index ac5cabff8..7f7ccc46f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRmlAdapterAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRmlAdapterAutoConfig.java index 813b8bf7f..c7e674cde 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRmlAdapterAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioRmlAdapterAutoConfig.java @@ -11,8 +11,8 @@ public class LdioRmlAdapterAutoConfig { @SuppressWarnings("java:S6830") - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter") - public LdioAdapterConfigurator ldiHttpOutConfigurator() { + @Bean("Ldio:RmlAdapter") + public LdioAdapterConfigurator ldioAdapterConfigurator() { return new LdioRmlAdapterProcessorConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index 8d454cf52..c3a3c8143 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstruct.java b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioSparqlConstruct.java similarity index 82% rename from ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstruct.java rename to ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioSparqlConstruct.java index 776837690..cbd2c3367 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstruct.java +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioSparqlConstruct.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; @@ -6,6 +6,7 @@ import org.apache.jena.rdf.model.Model; public class LdioSparqlConstruct extends LdioTransformer { + public static final String NAME = "Ldio:SparqlConstructTransformer"; private final SparqlConstructTransformer transformer; public LdioSparqlConstruct(Query query, boolean inferMode) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstructAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstructAutoConfig.java index 21711c06a..576e378b7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstructAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSparqlConstructAutoConfig.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioSparqlConstruct; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; @@ -8,10 +9,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioSparqlConstruct.NAME; + @Configuration public class LdioSparqlConstructAutoConfig { - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer") + @Bean(NAME) public LdioTransformerConfigurator ldioConfigurator() { return new LdioSparqlConstructTransformerConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index e81270285..27d24ab0e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiser.java b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionMaterialiser.java similarity index 84% rename from ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiser.java rename to ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionMaterialiser.java index 46d293714..83b7ecd33 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiser.java +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionMaterialiser.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; @@ -6,6 +6,7 @@ import org.apache.jena.rdf.model.Property; public class LdioVersionMaterialiser extends LdioTransformer { + public static final String NAME = "Ldio:VersionMaterialiser"; private final VersionMaterialiser versionMaterialiser; public LdioVersionMaterialiser(Property versionOfProperty, boolean restrictToMembers) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiserAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiserAutoConfig.java index 7730d2643..9d858dfc3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiserAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionMaterialiserAutoConfig.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionMaterialiser; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; @@ -9,9 +10,11 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionMaterialiser.NAME; + @Configuration public class LdioVersionMaterialiserAutoConfig { - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser") + @Bean(NAME) public LdioTransformerConfigurator ldioConfigurator() { return new LdioVersionMaterialiserTransformerConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index a427e0bc1..9b86c405d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreator.java b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java similarity index 78% rename from ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreator.java rename to ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java index a77c724de..72e7de1b0 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreator.java +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator; import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyExtractor; @@ -8,11 +8,11 @@ import org.apache.jena.rdf.model.Resource; public class LdioVersionObjectCreator extends LdioTransformer { - + public static final String NAME = "Ldio:VersionObjectCreator"; private final VersionObjectCreator versionObjectCreator; public LdioVersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, Resource memberType, - String delimiter, Property generatedAtProperty, Property versionOfProperty) { + String delimiter, Property generatedAtProperty, Property versionOfProperty) { this.versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter, generatedAtProperty, versionOfProperty); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java index e8a639d25..41104f7a7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.EmptyPropertyExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyPathExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionObjectCreator; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; @@ -15,9 +16,11 @@ import java.util.Optional; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionObjectCreator.NAME; + @Configuration public class LdioVersionObjectCreatorAutoConfig { - @Bean("be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator") + @Bean(NAME) public LdioTransformerConfigurator ldioConfigurator() { return new LdioVersionObjectCreatorTransformerConfigurator(); } diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index 6bed5a271..064d2cb1a 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 672863709..a2f415408 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 81ecd8d9d..b02de93a8 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 1.15.0-SNAPSHOT + 2.0.0-SNAPSHOT ldi-api