From 0bf0155cf0dfe4cf2aa69ce1677c04d10f147290 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:56:46 +0200 Subject: [PATCH] feat: include requester docs (#608) --- docs/_includes/ldio-core/http-requester.md | 93 ++++++++++--------- docs/_ldio/ldio-inputs/ldio-http-in-poller.md | 5 +- docs/_ldio/ldio-inputs/ldio-ldes-client.md | 8 +- 3 files changed, 54 insertions(+), 52 deletions(-) diff --git a/docs/_includes/ldio-core/http-requester.md b/docs/_includes/ldio-core/http-requester.md index 162ca2aaa..806ffbe2e 100644 --- a/docs/_includes/ldio-core/http-requester.md +++ b/docs/_includes/ldio-core/http-requester.md @@ -1,22 +1,51 @@ -### Additional LDIO Http Requester properties - -| Property | Description | Required | Default | Supported values | Example | -|:--------------------------|:----------------------------------------------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------------------------|:----------------------------| -| auth.type | The type of authentication required by the LDES server | No | NO_AUTH | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | OAUTH2_CLIENT_CREDENTIALS | -| auth.api-key | The api key when using auth.type 'API_KEY' | No | N/A | String | myKey | -| auth.api-key-header | The header for the api key when using auth.type 'API_KEY' | No | X-API-KEY | String | X-API-KEY | -| auth.client-id | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | String | myId | -| auth.client-secret | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | String | mySecret | -| auth.token-endpoint | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | HTTP and HTTPS urls | http://localhost:8000/token | -| auth.scope | The Oauth2 scope when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | HTTP and HTTPS urls | http://localhost:8000/token | -| retries.enabled | Indicates if the http client should retry http requests when the server cannot be reached. | No | true | true or false | true | -| retries.max | Max number of retries the http client should do when retries.enabled = true | No | 5 | Integer | 100 | -| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No | N/A | Comma seperated list of Integers | 410,451 | -| rate-limit.enabled | Indicates if the http client should limit http requests when calling the server. | No | false | true or false | false | -| rate-limit.limit | Limit of requests per period, which is defined below, that the http client should do when `rate-limit.enabled = true` | No | 500 | Integer | 100 | -| rate-limit.period | Period in which the limit of requests, which is defined above, can be reached by the http client when `rate-limit.enabled = true` | No | PT1M | ISO 8601 Duration | PT1H | -| http.headers.[].key/value | A list of custom http headers can be added. A key and value has to be provided for every header. | No | N/A | String | role | +### LDIO Http Requester properties +#### Authentication properties + +| Property | Description | Required | Default | Supported values | Example | +|:--------------------|:-----------------------------------------------------------------------|:---------|:----------|:----------------------------------------------|:----------------------------| +| auth.type | The type of authentication required by the LDES server | No | NO_AUTH | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | OAUTH2_CLIENT_CREDENTIALS | +| | | | | | | +| auth.api-key | The api key when using auth.type 'API_KEY' | No | N/A | String | myKey | +| auth.api-key-header | The header for the api key when using auth.type 'API_KEY' | No | X-API-KEY | String | X-API-KEY | +| | | | | | | +| auth.client-id | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | String | myId | +| auth.client-secret | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | String | mySecret | +| auth.token-endpoint | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | HTTP and HTTPS urls | http://localhost:8000/token | +| auth.scope | The Oauth2 scope when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | HTTP and HTTPS urls | http://localhost:8000/token | + +#### Retry properties + +| Property | Description | Required | Default | Supported values | Example | +|:--------------------------|:----------------------------------------------------------------------------------------------|:---------|:--------|:---------------------------------|:--------| +| retries.enabled | Indicates if the http client should retry http requests when the server cannot be reached. | No | true | Boolean value | true | +| retries.max | Max number of retries the http client should do when retries.enabled = true | No | 5 | Integer | 100 | +| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No | N/A | Comma seperated list of Integers | 410,451 | + +When retries are enabled, the following statuses are always retried, regardless of the configured statuses-to-retry: + +- 429 +- 5xx (500 and above) + +#### Rate limit properties + +| Property | Description | Required | Default | Supported values | Example | +|:-------------------|:----------------------------------------------------------------------------------------------------------------------------------|:---------|:--------|:------------------|:--------| +| rate-limit.enabled | Indicates if the http client should limit http requests when calling the server. | No | false | true or false | false | +| rate-limit.limit | Limit of requests per period, which is defined below, that the http client should do when `rate-limit.enabled = true` | No | 500 | Integer | 100 | +| rate-limit.period | Period in which the limit of requests, which is defined above, can be reached by the http client when `rate-limit.enabled = true` | No | PT1M | ISO 8601 Duration | PT1H | + +#### Http headers + +| Property | Description | Required | Default | Supported values | Example | +|:--------------------------|:-------------------------------------------------------------------------------------------------|:---------|:--------|:-----------------|:--------| +| http.headers.[].key/value | A list of custom http headers can be added. A key and value has to be provided for every header. | No | N/A | String | role | + + +
+ Example Http Requester config + +
```yaml config: http: @@ -38,32 +67,8 @@ period: P1D limit: 1000 ``` +
-### Retry -When retries are enabled, the following statuses are always retried, regardless of the configured statuses-to-retry: - -- 5xx (500 and above) -- 429 +
-### Rate limiter - -> **NOTE**: Since 1.14.0, rate-limiter.max-requests-per-minute is deprecated, as this was too restrictive for -> configuring the rate limiter. Here is an example on how to convert the old config. -> -> *Old config:* -> ```yaml -> config: -> rate-limit: -> enabled: true -> max-requests-per-minute: 500 -> ``` -> -> *New config:* -> ```yaml -> config: -> rate-limit: -> enabled: true -> limit: 500 -> period: PT1M -> ``` \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md index de6c61c87..1b0b816f8 100644 --- a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md +++ b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md @@ -11,6 +11,7 @@ title: HTTP In Poller The LDIO Http In Poller is a basic Http Poller that will poll a target URL on a specified interval. ## Config +### General properties | Property | Description | Required | Default | Example | Supported values | |:---------------|:-------------------------------------------------------------------|:---------|:--------|:--------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -19,9 +20,9 @@ The LDIO Http In Poller is a basic Http Poller that will poll a target URL on a | interval | Polling interval declared in ISO 8601 format. | Yes[^1] | N/A | PT1S | ISO 8601 formatted String | | continueOnFail | Indicated if continue if polling results in failure | No | true | true | true or false | -This component uses the "LDIO Http Requester" to make the HTTP request. -Refer to [LDIO Http Requester](../ldio-core) for the config. +{% include ldio-core/http-requester.md %} +### Multiple urls The Http In Poller supports polling multiple endpoints. Example configuration: ```yaml diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index e3311b72f..fbc535a87 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -67,7 +67,7 @@ The Client offers different ways to persist state of the processed members: > relative URIs are not used, `application/n-quads` or even the binary format `application/rdf+protobuf` are better > options, as these formats are faster to parse. - +> **_NOTE:_** Setting the keep-state property to `true` makes it so that the state can not be deleted through the pipeline-management api ### Version materialisation properties @@ -76,12 +76,8 @@ The Client offers different ways to persist state of the processed members: | _materialisation.enabled_ | Indicates if the client should return state-objects (true) or version-objects (false) | No | false | true | true or false | | _materialisation.version-of-property_| Property that points to the versionOfPath | No | http://purl.org/dc/terms/isVersionOf | "http://purl.org/dc/terms/isVersionOf" | true or false | -This component uses the "LDIO Http Requester" to make the HTTP-request. -Refer to [LDIO Http Requester](../ldio-core/ldio-http-requester) for the configuration of that component. - -> **_NOTE:_** Setting the keep-state property to `true` makes it so that the state can not be deleted through the pipeline-management api - +{% include ldio-core/http-requester.md %} ### Postgres properties