Skip to content

Commit

Permalink
Feat: Use generic request executor for http pollerer and ldes-client (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
Tomvbe authored Sep 13, 2023
1 parent 7f1b9e5 commit 62ecf97
Show file tree
Hide file tree
Showing 13 changed files with 243 additions and 207 deletions.
4 changes: 2 additions & 2 deletions ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
<artifactId>request-executor</artifactId>
<groupId>be.vlaanderen.informatievlaanderen.ldes.ldio</groupId>
<artifactId>ldio-request-executor</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,9 @@ public class HttpInputPoller extends LdiInput {
private static final String CONTENT_TYPE = "Content-Type";

public HttpInputPoller(ComponentExecutor executor, LdiAdapter adapter, List<String> endpoints,
boolean continueOnFail) {
boolean continueOnFail, RequestExecutor requestExecutor) {
super(executor, adapter);
RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory();
this.requestExecutor = requestExecutorFactory.createNoAuthExecutor();
this.requestExecutor = requestExecutor;
this.requests = endpoints.stream().map(endpoint -> new Request(endpoint, RequestHeaders.empty())).toList();
this.continueOnFail = continueOnFail;
this.scheduler = Executors.newSingleThreadScheduledExecutor();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter;
import be.vlaanderen.informatievlaanderen.ldes.ldio.HttpInputPoller;
import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator;
import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier;
import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
Expand All @@ -17,6 +18,8 @@
@Configuration
public class HttpInputPollerAutoConfig {

private static final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier();

@Bean("be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller")
public HttpInputPollerConfigurator httpInputPollerConfigurator() {
return new HttpInputPollerConfigurator();
Expand All @@ -40,7 +43,8 @@ public HttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor,
+ " cannot have following value: " + pollingInterval);
}

HttpInputPoller httpInputPoller = new HttpInputPoller(executor, adapter, endpoints, continueOnFail);
var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties);
var httpInputPoller = new HttpInputPoller(executor, adapter, endpoints, continueOnFail, requestExecutor);
httpInputPoller.schedulePoller(seconds);

return httpInputPoller;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package be.vlaanderen.informatievlaanderen.ldes.ldio;

import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory;
import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor;
import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter;
import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.MissingHeaderException;
Expand Down Expand Up @@ -29,6 +31,7 @@ class HttpInputPollerTest {
private static final String CONTENT = "_:b0 <http://schema.org/name> \"Jane Doe\" .";
private static final String CONTENT_TYPE = "application/n-quads";
private HttpInputPoller httpInputPoller;
private static RequestExecutor noAuthExecutor = new RequestExecutorFactory().createNoAuthExecutor();

@BeforeEach
void setUp() {
Expand All @@ -38,7 +41,7 @@ void setUp() {
.thenReturn(Stream.of())
.thenReturn(Stream.of());

httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), true);
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor);
}

@Test
Expand All @@ -54,7 +57,7 @@ void testClientPolling() {
void whenPolling_andMissesHeader() {
stubFor(get(ENDPOINT).willReturn(ok().withBody(CONTENT)));

httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false);
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor);
Executable polling = () -> httpInputPoller.poll();

assertThrows(MissingHeaderException.class, polling);
Expand All @@ -78,7 +81,7 @@ void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldSti
String otherEndpoint = "/other-resource";
stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT)));
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint),
true);
true, noAuthExecutor);

httpInputPoller.poll();

Expand All @@ -93,7 +96,7 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse() {
String otherEndpoint = "/other-endpoint";
stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT)));
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint),
true);
true, noAuthExecutor);

httpInputPoller.schedulePoller(1);

Expand All @@ -118,7 +121,7 @@ void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling() {
void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling() {
stubFor(get(ENDPOINT).willReturn(forbidden()));

httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false);
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor);
httpInputPoller.schedulePoller(1);

Mockito.verify(adapter, after(2000).never()).apply(any());
Expand All @@ -128,7 +131,8 @@ void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling() {
@Test
void when_EndpointDoesNotExist_Then_NoDataIsSent() {
String wrongEndpoint = "/non-existing-resource";
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + wrongEndpoint), true);
httpInputPoller = new HttpInputPoller(executor, adapter, List.of(BASE_URL + wrongEndpoint), true,
noAuthExecutor);

httpInputPoller.poll();

Expand Down
6 changes: 6 additions & 0 deletions ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@
<artifactId>tree-node-supplier</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>be.vlaanderen.informatievlaanderen.ldes.ldio</groupId>
<artifactId>ldio-request-executor</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,4 @@ private LdioLdesClientProperties() {
public static final String POSTGRES_PASSWORD = "postgres.password";
public static final String POSTGRES_URL = "postgres.url";

public static final String RETRIES_ENABLED = "retries.enabled";
public static final String MAX_RETRIES = "retries.max";
public static final String STATUSES_TO_RETRY = "retries.statuses-to-retry";

// authorization properties
public static final String AUTH_TYPE = "auth.type";
public static final String API_KEY = "auth.api-key";
public static final String API_KEY_HEADER = "auth.api-key-header";
public static final String CLIENT_ID = "auth.client-id";
public static final String CLIENT_SECRET = "auth.client-secret";
public static final String TOKEN_ENDPOINT = "auth.token-endpoint";
}
Original file line number Diff line number Diff line change
@@ -1,79 +1,29 @@
package be.vlaanderen.informatievlaanderen.ldes.ldio.config;

import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory;
import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy;
import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor;
import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter;
import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent;
import be.vlaanderen.informatievlaanderen.ldes.ldio.LdesClientRunner;
import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient;
import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties;
import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator;
import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier;
import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties;
import ldes.client.treenodesupplier.domain.valueobject.StatePersistence;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;

import static be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy.NO_AUTH;
import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.*;

public class LdioLdesClientConfigurator implements LdioInputConfigurator {

public static final String DEFAULT_API_KEY_HEADER = "X-API-KEY";

private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory();
private final LdioRequestExecutorSupplier ldioRequestExecutorSupplier = new LdioRequestExecutorSupplier();
private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory();

@Override
public LdiComponent configure(LdiAdapter adapter, ComponentExecutor componentExecutor,
ComponentProperties properties) {
RequestExecutor requestExecutor = getRequestExecutorWithPossibleRetry(properties);
RequestExecutor requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties);
StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(properties);
LdesClientRunner ldesClientRunner = new LdesClientRunner(requestExecutor, properties, componentExecutor,
statePersistence);
return new LdioLdesClient(componentExecutor, ldesClientRunner);
}

protected RequestExecutor getRequestExecutorWithPossibleRetry(ComponentProperties props) {
final RequestExecutor requestExecutor = getRequestExecutor(props);
boolean retriesEnabled = props.getOptionalBoolean(RETRIES_ENABLED).orElse(Boolean.TRUE);
if (retriesEnabled) {
int maxRetries = props.getOptionalInteger(MAX_RETRIES).orElse(5);
List<Integer> statusesToRetry = props.getOptionalProperty(STATUSES_TO_RETRY)
.map(csv -> Stream.of(csv.split(",")).map(String::trim).map(Integer::parseInt).toList())
.orElse(new ArrayList<>());
return requestExecutorFactory.createRetryExecutor(requestExecutor, maxRetries, statusesToRetry);
} else {
return requestExecutor;
}
}

private RequestExecutor getRequestExecutor(ComponentProperties componentProperties) {
Optional<AuthStrategy> authentication = AuthStrategy
.from(componentProperties.getOptionalProperty(LdioLdesClientProperties.AUTH_TYPE)
.orElse(NO_AUTH.name()));
if (authentication.isPresent()) {
return switch (authentication.get()) {
case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor();
case API_KEY ->
requestExecutorFactory.createApiKeyExecutor(
componentProperties.getOptionalProperty(LdioLdesClientProperties.API_KEY_HEADER)
.orElse(DEFAULT_API_KEY_HEADER),
componentProperties.getProperty(LdioLdesClientProperties.API_KEY));
case OAUTH2_CLIENT_CREDENTIALS ->
requestExecutorFactory.createClientCredentialsExecutor(
componentProperties.getProperty(LdioLdesClientProperties.CLIENT_ID),
componentProperties.getProperty(LdioLdesClientProperties.CLIENT_SECRET),
componentProperties.getProperty(LdioLdesClientProperties.TOKEN_ENDPOINT));
};
}
throw new UnsupportedOperationException(
"Requested authentication not available: "
+ componentProperties.getOptionalProperty(AUTH_TYPE).orElse("No auth type provided"));
}

}

This file was deleted.

Loading

0 comments on commit 62ecf97

Please sign in to comment.