From 62ecf979f71d8bd5e413dc3558983baa87ca231d Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:17:32 +0200 Subject: [PATCH] Feat: Use generic request executor for http pollerer and ldes-client (#333) --- .../ldio-http-in-poller/pom.xml | 4 +- .../ldes/ldio/HttpInputPoller.java | 5 +- .../config/HttpInputPollerAutoConfig.java | 6 +- .../ldes/ldio/HttpInputPollerTest.java | 16 ++- .../ldio-connectors/ldio-ldes-client/pom.xml | 6 + .../ldes/ldio/LdioLdesClientProperties.java | 11 -- .../config/LdioLdesClientConfigurator.java | 56 +------- .../config/LdioLdesClientAutoConfigTest.java | 131 ------------------ .../ldio-request-executor/pom.xml | 34 +++++ .../LdioRequestExecutorSupplier.java | 71 ++++++++++ .../RequestExecutorProperties.java | 20 +++ .../LdioRequestExecutorSupplierTest.java | 89 ++++++++++++ ldi-orchestrator/ldio-connectors/pom.xml | 1 + 13 files changed, 243 insertions(+), 207 deletions(-) delete mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfigTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml create mode 100644 ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index d0b9ce9d9..392f53387 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -23,8 +23,8 @@ test - be.vlaanderen.informatievlaanderen.ldes.ldi - request-executor + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-request-executor ${project.version} compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java index a2ad76eb2..6c1bca25c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPoller.java @@ -32,10 +32,9 @@ public class HttpInputPoller extends LdiInput { private static final String CONTENT_TYPE = "Content-Type"; public HttpInputPoller(ComponentExecutor executor, LdiAdapter adapter, List endpoints, - boolean continueOnFail) { + boolean continueOnFail, RequestExecutor requestExecutor) { super(executor, adapter); - RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); - this.requestExecutor = requestExecutorFactory.createNoAuthExecutor(); + this.requestExecutor = requestExecutor; this.requests = endpoints.stream().map(endpoint -> new Request(endpoint, RequestHeaders.empty())).toList(); this.continueOnFail = continueOnFail; this.scheduler = Executors.newSingleThreadScheduledExecutor(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java index 67a5b1da9..f70fbd577 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HttpInputPollerAutoConfig.java @@ -4,6 +4,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.HttpInputPoller; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,6 +18,8 @@ @Configuration public class HttpInputPollerAutoConfig { + private static final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier(); + @Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller") public HttpInputPollerConfigurator httpInputPollerConfigurator() { return new HttpInputPollerConfigurator(); @@ -40,7 +43,8 @@ public HttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, + " cannot have following value: " + pollingInterval); } - HttpInputPoller httpInputPoller = new HttpInputPoller(executor, adapter, endpoints, continueOnFail); + var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); + var httpInputPoller = new HttpInputPoller(executor, adapter, endpoints, continueOnFail, requestExecutor); httpInputPoller.schedulePoller(seconds); return httpInputPoller; diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java index 72b913ff9..c3d9fa472 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/HttpInputPollerTest.java @@ -1,5 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.MissingHeaderException; @@ -29,6 +31,7 @@ class HttpInputPollerTest { private static final String CONTENT = "_:b0 \"Jane Doe\" ."; private static final String CONTENT_TYPE = "application/n-quads"; private HttpInputPoller httpInputPoller; + private static RequestExecutor noAuthExecutor = new RequestExecutorFactory().createNoAuthExecutor(); @BeforeEach void setUp() { @@ -38,7 +41,7 @@ void setUp() { .thenReturn(Stream.of()) .thenReturn(Stream.of()); - httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), true); + httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor); } @Test @@ -54,7 +57,7 @@ void testClientPolling() { void whenPolling_andMissesHeader() { stubFor(get(ENDPOINT).willReturn(ok().withBody(CONTENT))); - httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false); + httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); Executable polling = () -> httpInputPoller.poll(); assertThrows(MissingHeaderException.class, polling); @@ -78,7 +81,7 @@ void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldSti String otherEndpoint = "/other-resource"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), - true); + true, noAuthExecutor); httpInputPoller.poll(); @@ -93,7 +96,7 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse() { String otherEndpoint = "/other-endpoint"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), - true); + true, noAuthExecutor); httpInputPoller.schedulePoller(1); @@ -118,7 +121,7 @@ void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling() { void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling() { stubFor(get(ENDPOINT).willReturn(forbidden())); - httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false); + httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor); httpInputPoller.schedulePoller(1); Mockito.verify(adapter, after(2000).never()).apply(any()); @@ -128,7 +131,8 @@ void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling() { @Test void when_EndpointDoesNotExist_Then_NoDataIsSent() { String wrongEndpoint = "/non-existing-resource"; - httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + wrongEndpoint), true); + httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + wrongEndpoint), true, + noAuthExecutor); httpInputPoller.poll(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 47a12e27e..636fa5f40 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -24,6 +24,12 @@ tree-node-supplier ${project.version} + + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-request-executor + ${project.version} + compile + org.springframework spring-test diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java index 21c092b3f..7e2356a28 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java @@ -15,15 +15,4 @@ private LdioLdesClientProperties() { public static final String POSTGRES_PASSWORD = "postgres.password"; public static final String POSTGRES_URL = "postgres.url"; - public static final String RETRIES_ENABLED = "retries.enabled"; - public static final String MAX_RETRIES = "retries.max"; - public static final String STATUSES_TO_RETRY = "retries.statuses-to-retry"; - - // authorization properties - public static final String AUTH_TYPE = "auth.type"; - public static final String API_KEY = "auth.api-key"; - public static final String API_KEY_HEADER = "auth.api-key-header"; - public static final String CLIENT_ID = "auth.client-id"; - public static final String CLIENT_SECRET = "auth.client-secret"; - public static final String TOKEN_ENDPOINT = "auth.token-endpoint"; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java index 2df7ac4a4..7e3b6ffed 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java @@ -1,79 +1,29 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdesClientRunner; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; -import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import static be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy.NO_AUTH; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.*; - public class LdioLdesClientConfigurator implements LdioInputConfigurator { - public static final String DEFAULT_API_KEY_HEADER = "X-API-KEY"; - - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier(); private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); @Override public LdiComponent configure(LdiAdapter adapter, ComponentExecutor componentExecutor, ComponentProperties properties) { - RequestExecutor requestExecutor = getRequestExecutorWithPossibleRetry(properties); + RequestExecutor requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(properties); LdesClientRunner ldesClientRunner = new LdesClientRunner(requestExecutor, properties, componentExecutor, statePersistence); return new LdioLdesClient(componentExecutor, ldesClientRunner); } - protected RequestExecutor getRequestExecutorWithPossibleRetry(ComponentProperties props) { - final RequestExecutor requestExecutor = getRequestExecutor(props); - boolean retriesEnabled = props.getOptionalBoolean(RETRIES_ENABLED).orElse(Boolean.TRUE); - if (retriesEnabled) { - int maxRetries = props.getOptionalInteger(MAX_RETRIES).orElse(5); - List statusesToRetry = props.getOptionalProperty(STATUSES_TO_RETRY) - .map(csv -> Stream.of(csv.split(",")).map(String::trim).map(Integer::parseInt).toList()) - .orElse(new ArrayList<>()); - return requestExecutorFactory.createRetryExecutor(requestExecutor, maxRetries, statusesToRetry); - } else { - return requestExecutor; - } - } - - private RequestExecutor getRequestExecutor(ComponentProperties componentProperties) { - Optional authentication = AuthStrategy - .from(componentProperties.getOptionalProperty(LdioLdesClientProperties.AUTH_TYPE) - .orElse(NO_AUTH.name())); - if (authentication.isPresent()) { - return switch (authentication.get()) { - case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(); - case API_KEY -> - requestExecutorFactory.createApiKeyExecutor( - componentProperties.getOptionalProperty(LdioLdesClientProperties.API_KEY_HEADER) - .orElse(DEFAULT_API_KEY_HEADER), - componentProperties.getProperty(LdioLdesClientProperties.API_KEY)); - case OAUTH2_CLIENT_CREDENTIALS -> - requestExecutorFactory.createClientCredentialsExecutor( - componentProperties.getProperty(LdioLdesClientProperties.CLIENT_ID), - componentProperties.getProperty(LdioLdesClientProperties.CLIENT_SECRET), - componentProperties.getProperty(LdioLdesClientProperties.TOKEN_ENDPOINT)); - }; - } - throw new UnsupportedOperationException( - "Requested authentication not available: " - + componentProperties.getOptionalProperty(AUTH_TYPE).orElse("No auth type provided")); - } - } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfigTest.java deleted file mode 100644 index d9e010734..000000000 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfigTest.java +++ /dev/null @@ -1,131 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.clientcredentials.ClientCredentialsRequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultRequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.retry.RetryExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Map; -import java.util.stream.Stream; - -import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -class LdioLdesClientAutoConfigTest { - private static final String ENDPOINT = "http://localhost:8080/endpoint"; - private static final String pipelineName = "pipeline"; - private LdioLdesClientConfigurator configurator; - @Mock - private LdiAdapter adapter; - - @Mock - private ComponentExecutor componentExecutor; - - @BeforeEach - void setUp() { - LdioLdesClientAutoConfig autoConfig = new LdioLdesClientAutoConfig(); - configurator = (LdioLdesClientConfigurator) autoConfig.ldioConfigurator(); - } - - @Test - void when_invalidConfigProvided_then_noInteractionsExpected() { - configurator.configure(adapter, componentExecutor, - new ComponentProperties(Map.of(PIPELINE_NAME, pipelineName))); - - verifyNoInteractions(componentExecutor); - } - - @ParameterizedTest - @ArgumentsSource(RequestExecutorConfigArgumentsProvider.class) - void when_validConfigProvided_then_requestExecutorCreated(ComponentProperties props, Class cls) { - LdioLdesClientConfigurator spyConfigurator = spy(configurator); - - spyConfigurator.configure(adapter, componentExecutor, props); - verify(spyConfigurator).getRequestExecutorWithPossibleRetry(props); - - RequestExecutor executor = configurator.getRequestExecutorWithPossibleRetry(props); - assertEquals(cls, executor.getClass()); - } - - @ParameterizedTest - @ValueSource(strings = { "api_key", "oauth2_client_credentials" }) - void when_autTypeProvided_and_additionalKeysAreMissing_then_throwException(String authType) { - ComponentProperties props = new ComponentProperties( - Map.of(PIPELINE_NAME, pipelineName, URL, ENDPOINT, AUTH_TYPE, authType)); - - assertThrows(IllegalArgumentException.class, () -> configurator.configure(adapter, componentExecutor, props)); - } - - @Test - void when_unsupportedAuthTypeProvided_then_throwException() { - final String INVALID_AUTH_TYPE = "invalid_auth_type"; - ComponentProperties props = new ComponentProperties( - Map.of(PIPELINE_NAME, pipelineName, URL, ENDPOINT, AUTH_TYPE, INVALID_AUTH_TYPE)); - - Exception e = assertThrows(UnsupportedOperationException.class, - () -> configurator.configure(adapter, componentExecutor, props)); - assertEquals("Requested authentication not available: " + INVALID_AUTH_TYPE, e.getMessage()); - } - - private static class RequestExecutorConfigArgumentsProvider implements ArgumentsProvider { - @Override - public Stream provideArguments(ExtensionContext extensionContext) { - return Stream.of( - Arguments.of(new ComponentProperties(Map.of(PIPELINE_NAME, pipelineName, RETRIES_ENABLED, "FALSE", - URL, ENDPOINT)), - DefaultRequestExecutor.class), - Arguments.of( - new ComponentProperties( - Map.of(PIPELINE_NAME, pipelineName, RETRIES_ENABLED, "false", URL, ENDPOINT, - AUTH_TYPE, "api_key", API_KEY, "my_secret_key")), - DefaultRequestExecutor.class), - Arguments.of( - new ComponentProperties(Map.of( - PIPELINE_NAME, pipelineName, - RETRIES_ENABLED, "false", - URL, ENDPOINT, - AUTH_TYPE, "oauth2_client_credentials", - CLIENT_ID, "client_id", - CLIENT_SECRET, "my_client_secret", - TOKEN_ENDPOINT, "http://localhost:8080/token-endpoint")), - ClientCredentialsRequestExecutor.class), - Arguments.of( - new ComponentProperties(Map.of(PIPELINE_NAME, pipelineName, URL, ENDPOINT, - RETRIES_ENABLED, "FALSE")), - DefaultRequestExecutor.class), - Arguments.of( - new ComponentProperties(Map.of(PIPELINE_NAME, pipelineName, URL, ENDPOINT, - RETRIES_ENABLED, "true")), - RetryExecutor.class), - Arguments.of( - new ComponentProperties(Map.of( - PIPELINE_NAME, pipelineName, - URL, ENDPOINT, - RETRIES_ENABLED, "TRUE", - MAX_RETRIES, "10", - AUTH_TYPE, "oauth2_client_credentials", - CLIENT_ID, "client_id", - CLIENT_SECRET, "my_client_secret", - TOKEN_ENDPOINT, "http://localhost:8080/token-endpoint")), - RetryExecutor.class)); - } - } -} diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml new file mode 100644 index 000000000..2f56f449f --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -0,0 +1,34 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-connectors + 1.7.0-SNAPSHOT + + + ldio-request-executor + + + + be.vlaanderen.informatievlaanderen.ldes.ldi + request-executor + ${project.version} + + + org.mockito + mockito-core + ${mockito-core.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito-junit.version} + test + + + + \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java new file mode 100644 index 000000000..a9c5211d6 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplier.java @@ -0,0 +1,71 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy.NO_AUTH; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.RequestExecutorProperties.*; + +/** + * Creates a RequestExecutor based on the config provided using LDIO + * ComponentProperties. + */ +public class LdioRequestExecutorSupplier { + + public static final String DEFAULT_API_KEY_HEADER = "X-API-KEY"; + + private final RequestExecutorFactory requestExecutorFactory; + + public LdioRequestExecutorSupplier() { + this(new RequestExecutorFactory()); + } + + public LdioRequestExecutorSupplier(RequestExecutorFactory requestExecutorFactory) { + this.requestExecutorFactory = requestExecutorFactory; + } + + public RequestExecutor getRequestExecutor(ComponentProperties props) { + final RequestExecutor requestExecutor = getAuthRequestExecutor(props); + boolean retriesEnabled = props.getOptionalBoolean(RETRIES_ENABLED).orElse(Boolean.TRUE); + if (retriesEnabled) { + int maxRetries = props.getOptionalInteger(MAX_RETRIES).orElse(5); + List statusesToRetry = props.getOptionalProperty(STATUSES_TO_RETRY) + .map(csv -> Stream.of(csv.split(",")).map(String::trim).map(Integer::parseInt).toList()) + .orElse(new ArrayList<>()); + return requestExecutorFactory.createRetryExecutor(requestExecutor, maxRetries, statusesToRetry); + } else { + return requestExecutor; + } + } + + private RequestExecutor getAuthRequestExecutor(ComponentProperties componentProperties) { + Optional authentication = AuthStrategy + .from(componentProperties.getOptionalProperty(AUTH_TYPE).orElse(NO_AUTH.name())); + if (authentication.isPresent()) { + return switch (authentication.get()) { + case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(); + case API_KEY -> + requestExecutorFactory + .createApiKeyExecutor( + componentProperties.getOptionalProperty(API_KEY_HEADER) + .orElse(DEFAULT_API_KEY_HEADER), + componentProperties.getProperty(API_KEY)); + case OAUTH2_CLIENT_CREDENTIALS -> + requestExecutorFactory.createClientCredentialsExecutor( + componentProperties.getProperty(CLIENT_ID), + componentProperties.getProperty(CLIENT_SECRET), + componentProperties.getProperty(TOKEN_ENDPOINT)); + }; + } + throw new UnsupportedOperationException("Requested authentication not available: " + + componentProperties.getOptionalProperty(AUTH_TYPE).orElse("No auth type provided")); + } + +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java new file mode 100644 index 000000000..524de7a57 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/RequestExecutorProperties.java @@ -0,0 +1,20 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor; + +public class RequestExecutorProperties { + + private RequestExecutorProperties() { + } + + public static final String RETRIES_ENABLED = "retries.enabled"; + public static final String MAX_RETRIES = "retries.max"; + public static final String STATUSES_TO_RETRY = "retries.statuses-to-retry"; + + // authorization properties + public static final String AUTH_TYPE = "auth.type"; + public static final String API_KEY = "auth.api-key"; + public static final String API_KEY_HEADER = "auth.api-key-header"; + public static final String CLIENT_ID = "auth.client-id"; + public static final String CLIENT_SECRET = "auth.client-secret"; + public static final String TOKEN_ENDPOINT = "auth.token-endpoint"; + +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java new file mode 100644 index 000000000..4d4f0ed92 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/requestexecutor/LdioRequestExecutorSupplierTest.java @@ -0,0 +1,89 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Map; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.RequestExecutorProperties.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class LdioRequestExecutorSupplierTest { + + @Mock + private RequestExecutorFactory requestExecutorFactory; + + @InjectMocks + private LdioRequestExecutorSupplier requestExecutorSupplier; + + @Test + void shouldReturnRetryExecutorWithDefaults_whenNoProperties() { + ComponentProperties properties = new ComponentProperties(Map.of()); + RequestExecutor requestExecutor = mock(RequestExecutor.class); + when(requestExecutorFactory.createNoAuthExecutor()).thenReturn(requestExecutor); + RequestExecutor retryRequestExecutor = mock(RequestExecutor.class); + when(requestExecutorFactory.createRetryExecutor(requestExecutor, 5, List.of())) + .thenReturn(retryRequestExecutor); + + RequestExecutor result = requestExecutorSupplier.getRequestExecutor(properties); + + assertEquals(retryRequestExecutor, result); + } + + @Test + void shouldReturnRetryExecutorWithConfiguredProperties_whenPropertiesConfigured() { + String maxRetries = "10"; + ComponentProperties properties = new ComponentProperties(Map.of( + MAX_RETRIES, maxRetries, + STATUSES_TO_RETRY, "400,404", + AUTH_TYPE, AuthStrategy.API_KEY.name(), + API_KEY_HEADER, "key-header", + API_KEY, "key")); + RequestExecutor requestExecutor = mock(RequestExecutor.class); + when(requestExecutorFactory.createApiKeyExecutor("key-header", "key")).thenReturn(requestExecutor); + RequestExecutor retryRequestExecutor = mock(RequestExecutor.class); + when(requestExecutorFactory.createRetryExecutor(requestExecutor, Integer.parseInt(maxRetries), + List.of(400, 404))) + .thenReturn(retryRequestExecutor); + + RequestExecutor result = requestExecutorSupplier.getRequestExecutor(properties); + + assertEquals(retryRequestExecutor, result); + } + + @Test + void shouldReturnNonRetryExecutorWithConfiguredProperties_whenPropertiesConfigured() { + ComponentProperties properties = new ComponentProperties(Map.of( + RETRIES_ENABLED, "false", + AUTH_TYPE, AuthStrategy.OAUTH2_CLIENT_CREDENTIALS.name(), + CLIENT_ID, "client", + CLIENT_SECRET, "secret", + TOKEN_ENDPOINT, "token")); + RequestExecutor requestExecutor = mock(RequestExecutor.class); + when(requestExecutorFactory.createClientCredentialsExecutor("client", "secret", "token")) + .thenReturn(requestExecutor); + + RequestExecutor result = requestExecutorSupplier.getRequestExecutor(properties); + + assertEquals(requestExecutor, result); + } + + @Test + void shouldThrowException_whenAuthTypeNotSupported() { + ComponentProperties properties = new ComponentProperties(Map.of(AUTH_TYPE, "fantasy")); + assertThrows(UnsupportedOperationException.class, + () -> requestExecutorSupplier.getRequestExecutor(properties)); + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index d3f3be2c0..10a281c46 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -34,6 +34,7 @@ ldio-azure-blob-out ldio-file-out ldio-repository-materialiser + ldio-request-executor