Merge pull request #2 from DiSSCo/hotfix/fix-properties

Hotfix/fix properties
DiSSCo · Nov 4, 2024 · 583b530 · 583b530
2 parents daa1375 + 7230786
commit 583b530
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 30 deletions.
diff --git a/src/main/java/eu/dissco/exportjob/component/JobRequestComponent.java b/src/main/java/eu/dissco/exportjob/component/JobRequestComponent.java
@@ -3,6 +3,7 @@
 import eu.dissco.exportjob.domain.JobRequest;
 import eu.dissco.exportjob.domain.JobStateEndpoint;
 import eu.dissco.exportjob.domain.SearchParam;
+import eu.dissco.exportjob.domain.TargetType;
 import eu.dissco.exportjob.exceptions.FailedProcessingException;
 import eu.dissco.exportjob.properties.JobProperties;
 import eu.dissco.exportjob.web.ExporterBackendClient;
@@ -29,9 +30,8 @@ public JobRequest getJobRequest() throws FailedProcessingException {
     for (int i = 0; i < properties.getInputFields().size(); i++) {
       searchParams.add(new SearchParam(properties.getInputFields().get(i), properties.getInputValues().get(i)));
     }
-    log.info("Received {} job request with id {} and {} search parameters", properties.getTargetType(), properties.getJobId(),
-        searchParams);
-    return new JobRequest(searchParams, properties.getTargetType(), properties.getJobId());
+    log.info("Received job request with id {} and {} search parameters", properties.getJobId(), searchParams);
+    return new JobRequest(searchParams, TargetType.fromString(properties.getTargetType()), properties.getJobId());
   }
 
 }
diff --git a/src/main/java/eu/dissco/exportjob/domain/TargetType.java b/src/main/java/eu/dissco/exportjob/domain/TargetType.java
@@ -1,16 +1,17 @@
 package eu.dissco.exportjob.domain;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.HashMap;
+import java.util.Map;
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 public enum TargetType {
 
-  @JsonProperty("https://doi.org/21.T11148/894b1e6cad57e921764e")
   DIGITAL_SPECIMEN(
       "https://doi.org/21.T11148/894b1e6cad57e921764e"),
 
-  @JsonProperty("https://doi.org/21.T11148/bbad8c4e101e8af01115")
-  MEDIA_OBJECT(
+  DIGITAL_MEDIA(
       "https://doi.org/21.T11148/bbad8c4e101e8af01115");
 
   @Getter
@@ -20,4 +21,14 @@ public enum TargetType {
     this.name = name;
   }
 
+  public static TargetType fromString(String s){
+    if ("https://doi.org/21.T11148/894b1e6cad57e921764e".equals(s)){
+      return DIGITAL_SPECIMEN;
+    } else if ("https://doi.org/21.T11148/bbad8c4e101e8af01115".equals(s)){
+      return DIGITAL_MEDIA;
+    }
+    log.error("Invalid target type {}", s);
+    throw new IllegalArgumentException();
+  }
+
 }
diff --git a/src/main/java/eu/dissco/exportjob/properties/JobProperties.java b/src/main/java/eu/dissco/exportjob/properties/JobProperties.java
@@ -1,7 +1,6 @@
 package eu.dissco.exportjob.properties;
 
-import eu.dissco.exportjob.domain.TargetType;
-import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
 import java.util.List;
 import java.util.UUID;
 import lombok.Data;
@@ -13,19 +12,20 @@
 @Validated
 @ConfigurationProperties(prefix = "job")
 public class JobProperties {
-  @NotBlank
+
+  @NotNull
   @Value("#{'${job.input-fields}'.split(',')}")
   List<String> inputFields;
 
-  @NotBlank
+  @NotNull
   @Value("#{'${job.input-values}'.split(',')}")
   List<String> inputValues;
 
-  @NotBlank
+  @NotNull
   @Value("${job.target-type}")
-  TargetType targetType;
+  String targetType;
 
-  @NotBlank
+  @NotNull
   @Value("${job.id}")
   UUID jobId;
 

diff --git a/src/main/java/eu/dissco/exportjob/properties/S3Properties.java b/src/main/java/eu/dissco/exportjob/properties/S3Properties.java
@@ -15,4 +15,7 @@ public class S3Properties {
 
   @NotBlank
   private String accessSecret;
+
+  @NotBlank
+  private String bucketName;
 }
diff --git a/src/main/java/eu/dissco/exportjob/repository/ElasticSearchRepository.java b/src/main/java/eu/dissco/exportjob/repository/ElasticSearchRepository.java
@@ -25,7 +25,7 @@ public class ElasticSearchRepository {
 
   private final ElasticsearchClient client;
   private final ElasticSearchProperties properties;
-  private static final String SORT_BY = "dcterms:identifier.keyword";
+  private static final String SORT_BY = "ods:ID.keyword";
 
   public List<JsonNode> getTargetObjects(List<SearchParam> searchParams, TargetType targetType,
       String lastId, List<String> targetFields)

diff --git a/src/main/java/eu/dissco/exportjob/repository/S3Repository.java b/src/main/java/eu/dissco/exportjob/repository/S3Repository.java
@@ -1,35 +1,39 @@
 package eu.dissco.exportjob.repository;
 
+import eu.dissco.exportjob.properties.S3Properties;
 import java.io.File;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
 import java.util.UUID;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Repository;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.transfer.s3.S3TransferManager;
 
+@Slf4j
 @Repository
 @RequiredArgsConstructor
 public class S3Repository {
 
   private final S3AsyncClient s3Client;
   private final DateTimeFormatter formatter;
-  private static final String BUCKET_NAME = "dissco-data-export";
+  private final S3Properties properties;
 
   public String uploadResults(File file, UUID jobId) {
+    log.info("Uploading results to S3");
     var key = getDate() +"/" + jobId.toString();
     try (var transferManager = S3TransferManager.builder().s3Client(s3Client).build()) {
       var upload = transferManager
           .uploadFile(uploadFileRequest -> uploadFileRequest
               .putObjectRequest(putObjectRequest -> putObjectRequest
-                  .bucket(BUCKET_NAME)
+                  .bucket(properties.getBucketName())
                   .key(key))
               .source(file));
       upload.completionFuture().join();
       return s3Client.utilities().getUrl(
               builder -> builder
-                  .bucket(BUCKET_NAME)
+                  .bucket(properties.getBucketName())
                   .key(key))
           .toString();
     }

diff --git a/src/main/java/eu/dissco/exportjob/service/AbstractExportJobService.java b/src/main/java/eu/dissco/exportjob/service/AbstractExportJobService.java
@@ -1,8 +1,5 @@
 package eu.dissco.exportjob.service;
 
-import static eu.dissco.exportjob.service.DoiListService.ID_FIELD;
-
-import co.elastic.clients.elasticsearch._types.query_dsl.FieldAndFormat;
 import com.fasterxml.jackson.databind.JsonNode;
 import eu.dissco.exportjob.domain.JobRequest;
 import eu.dissco.exportjob.domain.JobStateEndpoint;
@@ -26,7 +23,7 @@ public abstract class AbstractExportJobService {
   private final ExporterBackendClient exporterBackendClient;
   private final S3Repository s3Repository;
   protected static final String TEMP_FILE_NAME = "src/main/resources/tmp.csv.gz";
-  protected static final String ID_FIELD = "dcterms:identifier";
+  protected static final String ID_FIELD = "ods:ID";
   protected static final String PHYSICAL_ID_FIELD = "ods:physicalSpecimenID";
 
 
@@ -36,10 +33,12 @@ public void handleMessage(JobRequest jobRequest) throws FailedProcessingExceptio
       var uploadData = processSearchResults(jobRequest);
       if (uploadData) {
         var url = s3Repository.uploadResults(new File(TEMP_FILE_NAME), jobRequest.jobId());
+        log.info("Successfully posted results to s3 at url {}", url);
         exporterBackendClient.markJobAsComplete(jobRequest.jobId(), url);
       } else {
         exporterBackendClient.markJobAsComplete(jobRequest.jobId(), null);
       }
+      log.info("Successfully completed job {}", jobRequest.jobId());
     } catch (IOException e) {
       log.error("An error has occurred", e);
       exporterBackendClient.updateJobState(jobRequest.jobId(), JobStateEndpoint.FAILED);
@@ -50,20 +49,21 @@ private boolean processSearchResults(JobRequest jobRequest) throws IOException {
     String lastId = null;
     writeHeaderToFile();
     boolean keepSearching = true;
-    boolean resultsProcessed = false;
+    long resultsProcessed = 0L;
     var targetFields = targetFields();
     while (keepSearching) {
       var searchResult = elasticSearchRepository.getTargetObjects(jobRequest.searchParams(),
           jobRequest.targetType(), lastId, targetFields);
-      if (searchResult.isEmpty()){
+      if (searchResult.isEmpty()) {
         keepSearching = false;
       } else {
         writeResultsToFile(searchResult);
         lastId = searchResult.getLast().get(ID_FIELD).asText();
-        resultsProcessed = true;
+        resultsProcessed += searchResult.size();
       }
     }
-    return resultsProcessed;
+    log.info("Processed {} search results", resultsProcessed);
+    return resultsProcessed > 0;
   }
 
   protected abstract void writeHeaderToFile() throws IOException;

diff --git a/src/main/java/eu/dissco/exportjob/web/ExporterBackendClient.java b/src/main/java/eu/dissco/exportjob/web/ExporterBackendClient.java
@@ -49,7 +49,7 @@ public void markJobAsComplete(UUID jobId, String downloadLink) throws FailedProc
     try {
       webClient
           .method(HttpMethod.POST)
-          .uri(uriBuilder -> uriBuilder.path("/complete").build())
+          .uri(uriBuilder -> uriBuilder.path("/completed").build())
           .header("Authorization", "Bearer " + tokenAuthenticator.getToken())
           .body(BodyInserters.fromValue(body))
           .retrieve()

diff --git a/src/test/java/eu/dissco/exportjob/component/JobRequestComponentTest.java b/src/test/java/eu/dissco/exportjob/component/JobRequestComponentTest.java
@@ -47,7 +47,7 @@ void testHandleMessage() throws FailedProcessingException {
     properties.setInputFields(List.of(ORG_FIELD_NAME, ORG_FIELD_NAME));
     properties.setJobId(JOB_ID);
     properties.setInputValues(List.of(ORG_1, ORG_2));
-    properties.setTargetType(TargetType.DIGITAL_SPECIMEN);
+    properties.setTargetType(TargetType.DIGITAL_SPECIMEN.getName());
 
     // When
     var result = jobRequestComponent.getJobRequest();
@@ -62,7 +62,7 @@ void testHandleMessageInvalidParams() throws FailedProcessingException {
     properties.setInputFields(List.of(ORG_FIELD_NAME));
     properties.setJobId(JOB_ID);
     properties.setInputValues(List.of(ORG_1, ORG_2));
-    properties.setTargetType(TargetType.DIGITAL_SPECIMEN);
+    properties.setTargetType(TargetType.DIGITAL_SPECIMEN.getName());
 
     // When
     assertThrows(FailedProcessingException.class, () -> jobRequestComponent.getJobRequest());

diff --git a/src/test/java/eu/dissco/exportjob/domain/TargetTypeTest.java b/src/test/java/eu/dissco/exportjob/domain/TargetTypeTest.java
@@ -0,0 +1,25 @@
+package eu.dissco.exportjob.domain;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
+
+import org.junit.jupiter.api.Test;
+
+class TargetTypeTest {
+
+  @Test
+  void testFromStringMedia() {
+    // When
+    var type = TargetType.fromString("https://doi.org/21.T11148/bbad8c4e101e8af01115");
+
+    // Then
+    assertThat(type).isEqualTo(TargetType.DIGITAL_MEDIA);
+  }
+
+  @Test
+  void testInvalidFromString() {
+    // When / Then
+    assertThrowsExactly(IllegalArgumentException.class, () -> TargetType.fromString("bad type"));
+  }
+
+}
diff --git a/src/test/java/eu/dissco/exportjob/utils/TestUtils.java b/src/test/java/eu/dissco/exportjob/utils/TestUtils.java
@@ -25,7 +25,7 @@ private TestUtils() {
       .setSerializationInclusion(Include.NON_NULL);
   public static final String DOWNLOAD_LINK = "https://aws.download/s3";
   public static final String ORG_FIELD_NAME = "$['ods:organisationID']";
-  public static final String ID_FIELD = "dcterms:identifier";
+  public static final String ID_FIELD = "ods:ID";
   public static final String PHYS_ID_FIELD = "ods:physicalSpecimenID";
 
   public static JobRequest givenJobRequest() {