Skip to content

Commit

Permalink
Merge pull request #60 from DiSSCo/feature/opends-0.3.0
Browse files Browse the repository at this point in the history
Update to openDS 0.3.0
  • Loading branch information
samleeflang authored Jun 28, 2024
2 parents 0356a2f + 34c9943 commit 527c59b
Show file tree
Hide file tree
Showing 142 changed files with 1,339 additions and 2,460 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
-Dsonar.projectKey=DiSSCo_${{ github.event.repository.name }}
-Dsonar.exclusions=**/jooq/**
-Dsonar.coverage.exclusions=**/properties/**,**/configuration/**,**/exception/**
-Dsonar.coverage.exclusions=**/properties/**,**/configuration/**,**/exception/**,**/maven/**
- name: Login to Public ECR
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
Expand Down
27 changes: 22 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.2.5</version>
<version>3.3.1</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>eu.dissco.core</groupId>
Expand All @@ -19,11 +19,11 @@
<dwca-io.version>2.16</dwca-io.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<jaxb2-maven-plugin.version>3.1.0</jaxb2-maven-plugin.version>
<commons-compress.version>1.26.0</commons-compress.version>
<jakarta.activation-api.version>2.1.2</jakarta.activation-api.version>
<jakarta.xml.bind-api.version>4.0.1</jakarta.xml.bind-api.version>
<commons-compress.version>1.26.2</commons-compress.version>
<jakarta.activation-api.version>2.1.3</jakarta.activation-api.version>
<jakarta.xml.bind-api.version>4.0.2</jakarta.xml.bind-api.version>
<mockito-inline.version>5.2.0</mockito-inline.version>
<testcontainers.version>1.19.5</testcontainers.version>
<testcontainers.version>1.19.8</testcontainers.version>
<sonar.organization>dissco</sonar.organization>
<sonar.host.url>https://sonarcloud.io</sonar.host.url>
<sonar.coverage.jacoco.xmlReportPaths>../app-it/target/site/jacoco-aggregate/jacoco.xml
Expand Down Expand Up @@ -159,6 +159,11 @@
<artifactId>flyway-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-database-postgresql</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand All @@ -168,6 +173,18 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<mainClass>eu.dissco.core.translator.maven.MavenRunner</mainClass>
<arguments>
<argument>https://schemas.dissco.tech/schemas/fdo-type/digital-specimen/0.3.0/digital-specimen.json</argument>
<argument>https://schemas.dissco.tech/schemas/fdo-type/digital-media/0.3.0/digital-media.json</argument>
</arguments>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import java.time.Instant;
import java.util.Date;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

Expand All @@ -18,6 +19,7 @@ public ObjectMapper objectMapper() {
SimpleModule dateModule = new SimpleModule();
dateModule.addSerializer(Instant.class, new InstantSerializer());
dateModule.addDeserializer(Instant.class, new InstantDeserializer());
dateModule.addSerializer(Date.class, new DateSerializer());
mapper.registerModule(dateModule);
mapper.setSerializationInclusion(Include.NON_NULL);
return mapper;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package eu.dissco.core.translator.configuration;

import static eu.dissco.core.translator.configuration.ApplicationConfiguration.DATE_STRING;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import java.io.IOException;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class DateSerializer extends JsonSerializer<Date> {

private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DATE_STRING).withZone(
ZoneOffset.UTC);

@Override
public void serialize(Date value, JsonGenerator jsonGenerator,
SerializerProvider serializerProvider) {
try {
jsonGenerator.writeString(formatter.format(value.toInstant()));
} catch (IOException e) {
log.error("An error has occurred serializing a date. More information: {}", e.getMessage());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import java.util.List;

public record DigitalMediaObjectEvent(
public record DigitalMediaEvent(
List<String> enrichmentList,
DigitalMediaObject digitalMediaObject) {
DigitalMediaWrapper digitalMedia) {

}
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import eu.dissco.core.translator.schema.DigitalEntity;
import eu.dissco.core.translator.terms.specimen.PhysicalSpecimenId;
import eu.dissco.core.translator.terms.specimen.PhysicalSpecimenID;
import eu.dissco.core.translator.schema.DigitalMedia;

public record DigitalMediaObject(
public record DigitalMediaWrapper(
@JsonProperty("ods:type")
String type,
@JsonProperty(PhysicalSpecimenId.TERM)
String physicalSpecimenId,
@JsonProperty(PhysicalSpecimenID.TERM)
String physicalSpecimenID,
@JsonProperty("ods:attributes")
DigitalEntity attributes,
DigitalMedia attributes,
@JsonProperty("ods:originalAttributes")
JsonNode originalAttributes) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@
public record DigitalSpecimenEvent(
List<String> enrichmentList,
DigitalSpecimenWrapper digitalSpecimenWrapper,
List<DigitalMediaObjectEvent> digitalMediaObjectEvents) {
List<DigitalMediaEvent> digitalMediaEvents) {

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@


public record DigitalSpecimenWrapper(
@JsonProperty("ods:normalisedPhysicalSpecimenId")
@JsonProperty("ods:normalisedPhysicalSpecimenID")
String id,
@JsonProperty("ods:type")
String type,
Expand Down
58 changes: 58 additions & 0 deletions src/main/java/eu/dissco/core/translator/maven/MavenRunner.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package eu.dissco.core.translator.maven;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.RequestMethod;

public class MavenRunner {

private static final Logger LOGGER = LoggerFactory.getLogger(MavenRunner.class);

public static void main(String[] args) {
LOGGER.info("Starting the MavenRunner to download and parse json schemas");
for (String schemaUrl : args) {
LOGGER.info("Processing json schema: {}", schemaUrl);
var fileName = schemaUrl.substring(schemaUrl.lastIndexOf('/') + 1);
String outputFilePath = "src/main/resources/json-schema/" + fileName;
try {
String schema = downloadSchema(schemaUrl);
saveSchemaToFile(schema, outputFilePath);
LOGGER.info("JSON schema downloaded and saved to: {} ", outputFilePath);
} catch (IOException e) {
LOGGER.error("Error downloading or saving the JSON schema", e);
}
}
}

private static String downloadSchema(String schemaUrl) throws IOException {
StringBuilder result = new StringBuilder();
URL url = new URL(schemaUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod(RequestMethod.GET.name());
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
result.append(line).append("\n");
}
}
return result.toString();
}

private static void saveSchemaToFile(String schema, String filePath) throws IOException {
try (BufferedWriter writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(filePath), StandardCharsets.UTF_8))) {
writer.write(schema);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ public class FdoProperties {
private String digitalSpecimenType;

@NotBlank
private String digitalMediaObjectType;
private String digitalMediaType;

@NotBlank
private String applicationName = "DiSSCo Translator Service";

@NotBlank
private String applicationPID = "https://hdl.handle.net/TEST/XXX-XXX-XXX";

}
47 changes: 24 additions & 23 deletions src/main/java/eu/dissco/core/translator/service/BioCaseService.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
import eu.dissco.core.translator.Profiles;
import eu.dissco.core.translator.database.jooq.enums.JobState;
import eu.dissco.core.translator.domain.BioCasePartResult;
import eu.dissco.core.translator.domain.DigitalMediaObject;
import eu.dissco.core.translator.domain.DigitalMediaObjectEvent;
import eu.dissco.core.translator.domain.DigitalMediaWrapper;
import eu.dissco.core.translator.domain.DigitalMediaEvent;
import eu.dissco.core.translator.domain.DigitalSpecimenEvent;
import eu.dissco.core.translator.domain.DigitalSpecimenWrapper;
import eu.dissco.core.translator.domain.Enrichment;
Expand Down Expand Up @@ -187,29 +187,30 @@ private void processUnit(DataSet dataset, Unit unit, AtomicInteger processedReco
var unitAttributes = parseToJson(unit);
var datasetAttribute = getData(mapper.valueToTree(dataset.getMetadata()));
unitAttributes.setAll(datasetAttribute);
unitAttributes.put("abcd:datasetGUID", dataset.getDatasetGUID());
if (isAcceptedBasisOfRecord(unit)) {
try {
var attributes = digitalSpecimenDirector.assembleDigitalSpecimenTerm(unitAttributes, false);
if (attributes.getOdsNormalisedPhysicalSpecimenId() == null
|| attributes.getDwcInstitutionId() == null) {
if (attributes.getOdsNormalisedPhysicalSpecimenID() == null
|| attributes.getDwcInstitutionID() == null) {
throw new DiSSCoDataException(
"Record does not comply to MIDS level 0 (id and organisation), ignoring record");
}
var digitalSpecimen = new DigitalSpecimenWrapper(
attributes.getOdsNormalisedPhysicalSpecimenId(),
attributes.getOdsNormalisedPhysicalSpecimenID(),
fdoProperties.getDigitalSpecimenType(),
attributes,
cleanupRedundantFields(unitAttributes)
);
var digitalMediaObjects = processDigitalMediaObjects(
attributes.getOdsNormalisedPhysicalSpecimenId(), unit,
attributes.getDwcInstitutionId());
var digitalMedia = processDigitalMedia(
attributes.getOdsNormalisedPhysicalSpecimenID(), unit,
attributes.getDwcInstitutionID());
log.debug("Result digital Specimen: {}", digitalSpecimen);
kafkaService.sendMessage(
new DigitalSpecimenEvent(
enrichmentServices(false),
digitalSpecimen,
digitalMediaObjects));
digitalMedia));
processedRecords.incrementAndGet();
} catch (DiSSCoDataException e) {
log.error("Encountered data issue with record: {}", unitAttributes, e);
Expand Down Expand Up @@ -345,42 +346,42 @@ private JsonNode cleanupRedundantFields(JsonNode unitData) {
return data;
}

private List<DigitalMediaObjectEvent> processDigitalMediaObjects(String physicalSpecimenId,
private List<DigitalMediaEvent> processDigitalMedia(String physicalSpecimenId,
Unit unit, String organisationId) {
var digitalMediaObjectEvents = new ArrayList<DigitalMediaObjectEvent>();
var digitalMediaEvents = new ArrayList<DigitalMediaEvent>();
if (unit.getMultiMediaObjects() != null && !unit.getMultiMediaObjects().getMultiMediaObject()
.isEmpty()) {
for (MultiMediaObject media : unit.getMultiMediaObjects().getMultiMediaObject()) {
try {
digitalMediaObjectEvents.add(
processDigitalMediaObject(physicalSpecimenId, media, organisationId));
digitalMediaEvents.add(
processDigitalMedia(physicalSpecimenId, media, organisationId));
} catch (DiSSCoDataException e) {
log.error("Failed to process digital media object for digital specimen: {}",
physicalSpecimenId, e);
}
}
}
return digitalMediaObjectEvents;
return digitalMediaEvents;
}

private DigitalMediaObjectEvent processDigitalMediaObject(String physicalSpecimenId,
private DigitalMediaEvent processDigitalMedia(String physicalSpecimenId,
MultiMediaObject media, String organisationId) throws DiSSCoDataException {
var attributes = getData(mapper.valueToTree(media));
var digitalEntity = digitalSpecimenDirector.assembleDigitalMediaObjects(false, attributes,
var digitalMedia = digitalSpecimenDirector.assembleDigitalMedia(false, attributes,
organisationId);
if (digitalEntity.getAcAccessUri() == null) {
if (digitalMedia.getAcAccessURI() == null) {
throw new DiSSCoDataException(
"Digital media object for specimen does not have an access uri, ignoring record");
}
var digitalMediaObjectEvent = new DigitalMediaObjectEvent(enrichmentServices(true),
new DigitalMediaObject(
fdoProperties.getDigitalMediaObjectType(),
var digitalMediaEvent = new DigitalMediaEvent(enrichmentServices(true),
new DigitalMediaWrapper(
fdoProperties.getDigitalMediaType(),
physicalSpecimenId,
digitalEntity,
digitalMedia,
attributes
));
log.debug("Result digital media object: {}", digitalMediaObjectEvent);
return digitalMediaObjectEvent;
log.debug("Result digital media object: {}", digitalMediaEvent);
return digitalMediaEvent;
}

private List<String> enrichmentServices(boolean multiMediaObject) {
Expand Down
Loading

0 comments on commit 527c59b

Please sign in to comment.