Merge pull request #1 from DiSSCo/feature/init

init

pom.xml

@@ -35,6 +35,11 @@
     <elasticsearch.version>8.15.0</elasticsearch.version>
     <jakarta.json.version>2.1.3</jakarta.json.version>
     <opencsv.version>5.9</opencsv.version>
+    <amazon.awssdk.version>2.25.60</amazon.awssdk.version>
+    <sonar.organization>dissco</sonar.organization>
+    <sonar.host.url>https://sonarcloud.io</sonar.host.url>
+    <sonar.coverage.jacoco.xmlReportPaths>../app-it/target/site/jacoco-aggregate/jacoco.xml
+    </sonar.coverage.jacoco.xmlReportPaths>
   </properties>
   <dependencies>
     <dependency>
@@ -64,9 +69,21 @@
       <artifactId>opencsv</artifactId>
       <version>${opencsv.version}</version>
     </dependency>
-
-
-
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+      <version>${amazon.awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3-transfer-manager</artifactId>
+      <version>${amazon.awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>aws-crt-client</artifactId>
+      <version>${amazon.awssdk.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>elasticsearch</artifactId>
@@ -77,6 +94,21 @@
       <artifactId>spring-boot-starter-test</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>mockwebserver</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 
@@ -86,6 +118,38 @@
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-maven-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <version>0.8.12</version>
+        <executions>
+          <execution>
+            <id>prepare-agent</id>
+            <goals>
+              <goal>prepare-agent</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>prepare-agent-integration</id>
+            <goals>
+              <goal>prepare-agent-integration</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>report</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>report</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>report-integration</id>
+            <goals>
+              <goal>report-integration</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

src/main/java/eu/dissco/exportjob/DisscoExportJobApplication.java

@@ -0,0 +1,15 @@
+package eu.dissco.exportjob;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
+
+@SpringBootApplication
+@ConfigurationPropertiesScan
+public class DisscoExportJobApplication {
+
+  public static void main(String[] args) {
+    SpringApplication.run(DisscoExportJobApplication.class, args);
+  }
+
+}

src/main/java/eu/dissco/exportjob/Profiles.java

@@ -0,0 +1,7 @@
+package eu.dissco.exportjob;
+
+public class Profiles {
+  public static final String DOI_LIST = "doi_list";
+
+  private Profiles(){}
+}

src/main/java/eu/dissco/exportjob/ProjectRunner.java

@@ -0,0 +1,25 @@
+package eu.dissco.exportjob;
+
+import eu.dissco.exportjob.component.JobRequestComponent;
+import eu.dissco.exportjob.exceptions.FailedProcessingException;
+import eu.dissco.exportjob.service.AbstractExportJobService;
+import lombok.AllArgsConstructor;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.stereotype.Component;
+
+@Component
+@AllArgsConstructor
+public class ProjectRunner implements CommandLineRunner {
+
+  private final AbstractExportJobService exportJobService;
+  private final ConfigurableApplicationContext context;
+  private final JobRequestComponent jobRequestComponent;
+
+  @Override
+  public void run(String... args) throws FailedProcessingException {
+    var request = jobRequestComponent.getJobRequest();
+    exportJobService.handleMessage(request);
+    context.close();
+  }
+}

src/main/java/eu/dissco/exportjob/component/JobRequestComponent.java

@@ -0,0 +1,37 @@
+package eu.dissco.exportjob.component;
+
+import eu.dissco.exportjob.domain.JobRequest;
+import eu.dissco.exportjob.domain.JobStateEndpoint;
+import eu.dissco.exportjob.domain.SearchParam;
+import eu.dissco.exportjob.exceptions.FailedProcessingException;
+import eu.dissco.exportjob.properties.JobProperties;
+import eu.dissco.exportjob.web.ExporterBackendClient;
+import java.util.ArrayList;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+@Component
+@Slf4j
+@RequiredArgsConstructor
+public class JobRequestComponent {
+
+  private final JobProperties properties;
+  private final ExporterBackendClient client;
+
+  public JobRequest getJobRequest() throws FailedProcessingException {
+    var searchParams = new ArrayList<SearchParam>();
+    if (properties.getInputFields().size() != properties.getInputValues().size()) {
+      log.error("Mismatch between input fields and input values for searching");
+      client.updateJobState(properties.getJobId(), JobStateEndpoint.FAILED);
+      throw new FailedProcessingException();
+    }
+    for (int i = 0; i < properties.getInputFields().size(); i++) {
+      searchParams.add(new SearchParam(properties.getInputFields().get(i), properties.getInputValues().get(i)));
+    }
+    log.info("Received {} job request with id {} and {} search parameters", properties.getTargetType(), properties.getJobId(),
+        searchParams);
+    return new JobRequest(searchParams, properties.getTargetType(), properties.getJobId());
+  }
+
+}

src/main/java/eu/dissco/exportjob/configuration/ApplicationConfiguration.java

@@ -0,0 +1,27 @@
+package eu.dissco.exportjob.configuration;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import lombok.RequiredArgsConstructor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@RequiredArgsConstructor
+public class ApplicationConfiguration {
+
+  @Bean
+  public ObjectMapper objectMapper() {
+    var mapper = new ObjectMapper().findAndRegisterModules();
+    mapper.setSerializationInclusion(Include.NON_NULL);
+    return mapper;
+  }
+
+  @Bean
+  public DateTimeFormatter formatter() {
+    return DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC);
+  }
+
+}

src/main/java/eu/dissco/exportjob/configuration/ElasticSearchConfiguration.java

@@ -0,0 +1,40 @@
+package eu.dissco.exportjob.configuration;
+
+import eu.dissco.exportjob.properties.ElasticSearchProperties;
+
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.json.jackson.JacksonJsonpMapper;
+import co.elastic.clients.transport.ElasticsearchTransport;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.elasticsearch.client.RestClient;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@RequiredArgsConstructor
+public class ElasticSearchConfiguration {
+  private final ElasticSearchProperties properties;
+  private final ObjectMapper mapper;
+
+  @Bean
+  public ElasticsearchClient elasticsearchClient() {
+    var credentialsProvider = new BasicCredentialsProvider();
+    credentialsProvider.setCredentials(AuthScope.ANY,
+        new UsernamePasswordCredentials(properties.getUsername(), properties.getPassword()));
+    RestClient restClient = RestClient.builder(new HttpHost(properties.getHostname(),
+            properties.getPort()))
+        .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
+            .setDefaultCredentialsProvider(credentialsProvider)).build();
+    ElasticsearchTransport transport = new RestClientTransport(restClient,
+        new JacksonJsonpMapper(mapper));
+    return new ElasticsearchClient(transport);
+  }
+
+}

src/main/java/eu/dissco/exportjob/configuration/S3Configuration.java

@@ -0,0 +1,33 @@
+package eu.dissco.exportjob.configuration;
+
+import eu.dissco.exportjob.properties.S3Properties;
+import lombok.RequiredArgsConstructor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+
+@Configuration
+@RequiredArgsConstructor
+public class S3Configuration {
+  private final S3Properties s3Properties;
+
+  @Bean
+  public S3AsyncClient s3Client() {
+    return S3AsyncClient.crtBuilder()
+        .credentialsProvider(StaticCredentialsProvider.create(
+            AwsBasicCredentials.create(s3Properties.getAccessKey(),
+                s3Properties.getAccessSecret())))
+        .region(Region.EU_WEST_2)
+        .build();
+  }
+
+  @Bean
+  public S3TransferManager transferManager() {
+    return S3TransferManager.builder().s3Client(s3Client()).build();
+  }
+
+}

src/main/java/eu/dissco/exportjob/configuration/WebClientConfiguration.java

@@ -0,0 +1,46 @@
+package eu.dissco.exportjob.configuration;
+
+import jakarta.validation.constraints.NotBlank;
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
+import org.springframework.web.reactive.function.client.WebClient;
+import reactor.netty.http.client.HttpClient;
+
+
+@Configuration
+@RequiredArgsConstructor
+public class WebClientConfiguration {
+
+  @NotBlank
+  @Value("${endpoint.backend}")
+  private String backendEndpoint;
+
+  @NotBlank
+  @Value("${endpoint.token}")
+  private String tokenEndpoint;
+
+  @Bean(name= "exporterClient")
+  public WebClient webClient(){
+    return WebClient.builder()
+        .clientConnector(new ReactorClientHttpConnector(HttpClient.create()))
+        .baseUrl(backendEndpoint)
+        .defaultHeader(HttpHeaders.CONTENT_TYPE)
+        .build();
+  }
+
+  @Bean(name = "tokenClient")
+  public WebClient tokenClient() {
+    return WebClient.builder()
+        .clientConnector(new ReactorClientHttpConnector(HttpClient.create()))
+        .baseUrl(tokenEndpoint)
+        .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_FORM_URLENCODED_VALUE)
+        .build();
+  }
+
+
+}

src/main/java/eu/dissco/exportjob/domain/JobRequest.java

@@ -0,0 +1,12 @@
+package eu.dissco.exportjob.domain;
+
+import java.util.List;
+import java.util.UUID;
+
+public record JobRequest(
+    List<SearchParam> searchParams,
+    TargetType targetType,
+    UUID jobId
+) {
+
+}

src/main/java/eu/dissco/exportjob/domain/JobStateEndpoint.java

@@ -0,0 +1,14 @@
+package eu.dissco.exportjob.domain;
+
+import lombok.Getter;
+
+@Getter
+public enum JobStateEndpoint {
+  FAILED("/failed"),
+  RUNNING("/running");
+
+  private final String endpoint;
+  JobStateEndpoint(String s){
+    this.endpoint = s;
+  }
+}

src/main/java/eu/dissco/exportjob/domain/JobType.java

@@ -0,0 +1,7 @@
+package eu.dissco.exportjob.domain;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public enum JobType {
+  @JsonProperty("doi_list") DOI_LIST
+}

src/main/java/eu/dissco/exportjob/domain/SearchParam.java

@@ -0,0 +1,8 @@
+package eu.dissco.exportjob.domain;
+
+public record SearchParam(
+    String inputField,
+    String inputValue
+) {
+
+}