Skip to content

Commit

Permalink
Merge pull request #64 from DiSSCo/feature/add-source-system-name
Browse files Browse the repository at this point in the history
Add source system name to digital specimen and digital media
  • Loading branch information
samleeflang authored Sep 2, 2024
2 parents 7ea185c + 3809282 commit 1815758
Show file tree
Hide file tree
Showing 21 changed files with 171 additions and 102 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM eclipse-temurin:17-jdk-alpine as builder
FROM eclipse-temurin:17-jdk-alpine AS builder
WORKDIR application
ARG JAR_FILE=target/*.jar
COPY ${JAR_FILE} application.jar
Expand Down
6 changes: 3 additions & 3 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.3.1</version>
<version>3.3.3</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>eu.dissco.core</groupId>
Expand All @@ -19,11 +19,11 @@
<dwca-io.version>2.16</dwca-io.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<jaxb2-maven-plugin.version>3.1.0</jaxb2-maven-plugin.version>
<commons-compress.version>1.26.2</commons-compress.version>
<commons-compress.version>1.27.1</commons-compress.version>
<jakarta.activation-api.version>2.1.3</jakarta.activation-api.version>
<jakarta.xml.bind-api.version>4.0.2</jakarta.xml.bind-api.version>
<mockito-inline.version>5.2.0</mockito-inline.version>
<testcontainers.version>1.19.8</testcontainers.version>
<testcontainers.version>1.20.1</testcontainers.version>
<sonar.organization>dissco</sonar.organization>
<sonar.host.url>https://sonarcloud.io</sonar.host.url>
<sonar.coverage.jacoco.xmlReportPaths>../app-it/target/site/jacoco-aggregate/jacoco.xml
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package eu.dissco.core.translator.component;

import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.repository.SourceSystemRepository;
import lombok.Getter;
import org.springframework.stereotype.Component;

@Getter
@Component
public class SourceSystemComponent {

private final String sourceSystemID;
private final String sourceSystemName;
private final String sourceSystemEndpoint;

public SourceSystemComponent(WebClientProperties webClientProperties,
SourceSystemRepository repository) {
this.sourceSystemID = webClientProperties.getSourceSystemId();
var sourceSystemInformation = repository.getSourceSystem(this.sourceSystemID);
if (sourceSystemInformation == null) {
throw new IllegalArgumentException("Source System Identifier: " + sourceSystemID + " not found");
}
this.sourceSystemName = sourceSystemInformation.sourceSystemName();
this.sourceSystemEndpoint = sourceSystemInformation.sourceSystemUrl();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package eu.dissco.core.translator.domain;

public record SourceSystemInformation(String sourceSystemName, String sourceSystemUrl) {

}
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

import static eu.dissco.core.translator.database.jooq.Tables.SOURCE_SYSTEM;

import eu.dissco.core.translator.domain.SourceSystemInformation;
import lombok.RequiredArgsConstructor;
import org.jooq.DSLContext;
import org.jooq.Record1;
import org.jooq.Record2;
import org.springframework.stereotype.Repository;

@Repository
Expand All @@ -13,10 +14,15 @@ public class SourceSystemRepository {

private final DSLContext context;

public String getEndpoint(String sourceSystemId) {
return context.select(SOURCE_SYSTEM.ENDPOINT)
public SourceSystemInformation getSourceSystem(String sourceSystemId) {
return context.select(SOURCE_SYSTEM.NAME, SOURCE_SYSTEM.ENDPOINT)
.from(SOURCE_SYSTEM)
.where(SOURCE_SYSTEM.ID.eq(sourceSystemId))
.fetchOne(Record1::value1);
.fetchOne(this::mapToSourceSystemInformation);
}

private SourceSystemInformation mapToSourceSystemInformation(
Record2<String, String> dbRecord) {
return new SourceSystemInformation(dbRecord.value1(), dbRecord.value2());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import efg.MultiMediaObject;
import efg.Unit;
import eu.dissco.core.translator.Profiles;
import eu.dissco.core.translator.component.SourceSystemComponent;
import eu.dissco.core.translator.database.jooq.enums.JobState;
import eu.dissco.core.translator.domain.BioCasePartResult;
import eu.dissco.core.translator.domain.DigitalMediaEvent;
Expand All @@ -26,7 +27,6 @@
import eu.dissco.core.translator.properties.EnrichmentProperties;
import eu.dissco.core.translator.properties.FdoProperties;
import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.repository.SourceSystemRepository;
import eu.dissco.core.translator.terms.BaseDigitalObjectDirector;
import freemarker.template.Configuration;
import freemarker.template.TemplateException;
Expand Down Expand Up @@ -76,7 +76,7 @@ public class BioCaseService extends WebClientService {
private final ObjectMapper mapper;
private final WebClientProperties webClientProperties;
private final WebClient webClient;
private final SourceSystemRepository repository;
private final SourceSystemComponent sourceSystemComponent;
private final Configuration configuration;
private final XMLInputFactory xmlFactory;
private final KafkaService kafkaService;
Expand All @@ -94,7 +94,7 @@ private boolean isAcceptedBasisOfRecord(Unit unit) {

@Override
public TranslatorJobResult retrieveData() {
var uri = repository.getEndpoint(webClientProperties.getSourceSystemId());
var uri = sourceSystemComponent.getSourceSystemEndpoint();
var templateProperties = getTemplateProperties();
configuration.setNumberFormat("computer");
var finished = false;
Expand Down
10 changes: 4 additions & 6 deletions src/main/java/eu/dissco/core/translator/service/DwcaService.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import eu.dissco.core.translator.Profiles;
import eu.dissco.core.translator.component.SourceSystemComponent;
import eu.dissco.core.translator.database.jooq.enums.JobState;
import eu.dissco.core.translator.domain.DigitalMediaEvent;
import eu.dissco.core.translator.domain.DigitalMediaWrapper;
Expand All @@ -18,9 +19,7 @@
import eu.dissco.core.translator.properties.DwcaProperties;
import eu.dissco.core.translator.properties.EnrichmentProperties;
import eu.dissco.core.translator.properties.FdoProperties;
import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.repository.DwcaRepository;
import eu.dissco.core.translator.repository.SourceSystemRepository;
import eu.dissco.core.translator.terms.BaseDigitalObjectDirector;
import java.io.File;
import java.io.FileInputStream;
Expand Down Expand Up @@ -73,12 +72,11 @@ public class DwcaService extends WebClientService {
private static final String EXTENSIONS = "extensions";

private final ObjectMapper mapper;
private final WebClientProperties webClientProperties;
private final WebClient webClient;
private final DwcaProperties dwcaProperties;
private final KafkaService kafkaService;
private final EnrichmentProperties enrichmentProperties;
private final SourceSystemRepository repository;
private final SourceSystemComponent sourceSystemComponent;
private final DwcaRepository dwcaRepository;
private final BaseDigitalObjectDirector digitalSpecimenDirector;
private final FdoProperties fdoProperties;
Expand All @@ -88,7 +86,7 @@ public class DwcaService extends WebClientService {

@Override
public TranslatorJobResult retrieveData() {
var endpoint = repository.getEndpoint(webClientProperties.getSourceSystemId());
var endpoint = sourceSystemComponent.getSourceSystemEndpoint();
Archive archive = null;
var processedRecords = new AtomicInteger(0);
try {
Expand Down Expand Up @@ -386,7 +384,7 @@ private List<String> generateTableNames(Archive archive) {
}

private String getTableName(ArchiveFile archiveFile) {
var fullSourceSystemId = webClientProperties.getSourceSystemId();
var fullSourceSystemId = sourceSystemComponent.getSourceSystemID();
var minifiedSourceSystemId = fullSourceSystemId.substring(fullSourceSystemId.indexOf('/') + 1)
.replace("-", "_");
var tableName = "temp_" + (minifiedSourceSystemId + "_" + archiveFile.getRowType()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dissco.core.translator.component.OrganisationNameComponent;
import eu.dissco.core.translator.component.SourceSystemComponent;
import eu.dissco.core.translator.exception.OrganisationException;
import eu.dissco.core.translator.exception.UnknownPhysicalSpecimenIdType;
import eu.dissco.core.translator.properties.FdoProperties;
import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.schema.Agent;
import eu.dissco.core.translator.schema.Citation;
import eu.dissco.core.translator.schema.DigitalMedia;
Expand Down Expand Up @@ -210,7 +210,7 @@ public abstract class BaseDigitalObjectDirector {
protected final ObjectMapper mapper;
protected final TermMapper termMapper;
private final OrganisationNameComponent organisationNameComponent;
private final WebClientProperties webClientProperties;
private final SourceSystemComponent sourceSystemComponent;
private final FdoProperties fdoProperties;
private final List<String> identifierTerms;

Expand Down Expand Up @@ -280,14 +280,14 @@ private DigitalSpecimen assembleDigitalSpecimenTerms(JsonNode data, boolean dwc)
.withOdsPhysicalSpecimenID(physicalSpecimenId)
.withOdsIsKnownToContainMedia(parseToBoolean(new HasMedia(), data, dwc))
.withOdsSourceSystemID(
"https://hdl.handle.net/" + webClientProperties.getSourceSystemId())
"https://hdl.handle.net/" + sourceSystemComponent.getSourceSystemID())
.withOdsSourceSystemName(sourceSystemComponent.getSourceSystemName())
.withOdsLivingOrPreserved(
retrieveEnum(new LivingOrPreserved(), data, dwc, OdsLivingOrPreserved.class))
.withDwcPreparations(termMapper.retrieveTerm(new Preparations(), data, dwc))
.withDwcCollectionID(termMapper.retrieveTerm(new CollectionID(), data, dwc))
.withDctermsModified(termMapper.retrieveTerm(new Modified(), data, dwc))
.withOdsOrganisationName(
getOrganisationName(organisationId))
.withOdsOrganisationName(getOrganisationName(organisationId))
.withDwcRecordedBy(termMapper.retrieveTerm(new RecordedBy(), data, dwc))
.withDwcRecordedByID(termMapper.retrieveTerm(new RecordedByID(), data, dwc))
.withDwcBasisOfRecord(termMapper.retrieveTerm(new BasisOfRecord(), data, dwc))
Expand Down Expand Up @@ -613,7 +613,8 @@ public DigitalMedia assembleDigitalMedia(boolean dwc, JsonNode mediaRecord,
.withOdsStatus(DigitalMedia.OdsStatus.ODS_ACTIVE)
.withOdsType(fdoProperties.getDigitalMediaType())
.withOdsSourceSystemID(
"https://hdl.handle.net/" + webClientProperties.getSourceSystemId())
"https://hdl.handle.net/" + sourceSystemComponent.getSourceSystemID())
.withOdsSourceSystemName(sourceSystemComponent.getSourceSystemName())
.withOdsOrganisationID(organisationId)
.withOdsOrganisationName(getOrganisationName(organisationId))
.withAcAccessURI(termMapper.retrieveTerm(new AccessURI(), mediaRecord, dwc))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dissco.core.translator.Profiles;
import eu.dissco.core.translator.component.OrganisationNameComponent;
import eu.dissco.core.translator.component.SourceSystemComponent;
import eu.dissco.core.translator.properties.FdoProperties;
import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.schema.Citation;
import eu.dissco.core.translator.schema.DigitalSpecimen;
import eu.dissco.core.translator.schema.Identification;
Expand All @@ -22,9 +22,9 @@
public class BiocaseDigitalObjectDirector extends BaseDigitalObjectDirector {

public BiocaseDigitalObjectDirector(ObjectMapper mapper, TermMapper termMapper,
OrganisationNameComponent rorComponent, WebClientProperties webClientProperties,
OrganisationNameComponent rorComponent, SourceSystemComponent sourceSystemComponent,
FdoProperties fdoProperties) {
super(mapper, termMapper, rorComponent, webClientProperties, fdoProperties, identifierTerms());
super(mapper, termMapper, rorComponent, sourceSystemComponent, fdoProperties, identifierTerms());
}

private static List<String> identifierTerms() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dissco.core.translator.Profiles;
import eu.dissco.core.translator.component.OrganisationNameComponent;
import eu.dissco.core.translator.component.SourceSystemComponent;
import eu.dissco.core.translator.properties.FdoProperties;
import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.schema.Citation;
import eu.dissco.core.translator.schema.DigitalSpecimen;
import eu.dissco.core.translator.schema.Identification;
Expand All @@ -23,9 +23,9 @@ public class DwcaDigitalObjectDirector extends BaseDigitalObjectDirector {
private static final String EXTENSION = "extensions";

public DwcaDigitalObjectDirector(ObjectMapper mapper, TermMapper termMapper,
OrganisationNameComponent rorComponent, WebClientProperties webClientProperties,
OrganisationNameComponent rorComponent, SourceSystemComponent sourceSystemComponent,
FdoProperties fdoProperties) {
super(mapper, termMapper, rorComponent, webClientProperties, fdoProperties, identifierTerms());
super(mapper, termMapper, rorComponent, sourceSystemComponent, fdoProperties, identifierTerms());
}

private static List<String> identifierTerms() {
Expand Down
11 changes: 0 additions & 11 deletions src/main/java/eu/dissco/core/translator/terms/SourceSystemId.java

This file was deleted.

7 changes: 7 additions & 0 deletions src/main/resources/json-schema/digital-media.json
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,13 @@
"description": "The handle to the source system object which retrieved the data from the CMS",
"pattern": "^https:\/\/hdl\\.handle\\.net\/[\\w.]+\/(.){3}-(.){3}-(.){3}"
},
"ods:sourceSystemName": {
"type": "string",
"description": "The name of the source system as provided to DiSSCo",
"examples": [
"Naturalis Biodiversity Center (NL) - Vermes"
]
},
"ods:organisationID": {
"type": "string",
"description": "ROR or Wikidata identifie of the organisation",
Expand Down
9 changes: 8 additions & 1 deletion src/main/resources/json-schema/digital-specimen.json
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,13 @@
"description": "The handle to the source system object which retrieved the data from the CMS",
"pattern": "^https:\/\/hdl\\.handle\\.net\/[\\w.]+\/(.){3}-(.){3}-(.){3}"
},
"ods:sourceSystemName": {
"type": "string",
"description": "The name of the source system as provided to DiSSCo",
"examples": [
"Naturalis Biodiversity Center (NL) - Vermes"
]
},
"ods:livingOrPreserved": {
"description": "Whether the specimen is living or preserved",
"enum": [
Expand Down Expand Up @@ -443,4 +450,4 @@
"ods:organisationID"
],
"additionalProperties": false
}
}
6 changes: 6 additions & 0 deletions src/test/java/eu/dissco/core/translator/TestUtils.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package eu.dissco.core.translator;

import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dissco.core.translator.domain.SourceSystemInformation;
import eu.dissco.core.translator.schema.DigitalMedia;
import eu.dissco.core.translator.schema.DigitalSpecimen;
import java.io.IOException;
Expand All @@ -15,6 +16,7 @@ public class TestUtils {

public static final ObjectMapper MAPPER = new ObjectMapper().findAndRegisterModules();
public static final String SOURCE_SYSTEM_ID = "20.5000.1025/GW0-TYL-YRU";
public static final String SOURCE_SYSTEM_NAME = "Naturalis Biodiversity Center (NL) - Vermes";
public static final String ENDPOINT = "https://data.rbge.org.uk/service/dwca/data/darwin_core_living.zip";

public static final String ORGANISATION_ID = "https://ror.org/02y22ws83";
Expand Down Expand Up @@ -101,4 +103,8 @@ public static Stream<Arguments> provideInvalidDigitalSpecimen() {
Arguments.of(new DigitalSpecimen().withOdsOrganisationID(ORGANISATION_ID))
);
}

public static SourceSystemInformation givenSourceSystemInformation() {
return new SourceSystemInformation(SOURCE_SYSTEM_NAME, ENDPOINT);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package eu.dissco.core.translator.component;

import static eu.dissco.core.translator.TestUtils.ENDPOINT;
import static eu.dissco.core.translator.TestUtils.SOURCE_SYSTEM_ID;
import static eu.dissco.core.translator.TestUtils.SOURCE_SYSTEM_NAME;
import static eu.dissco.core.translator.TestUtils.givenSourceSystemInformation;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.BDDMockito.given;

import eu.dissco.core.translator.properties.WebClientProperties;
import eu.dissco.core.translator.repository.SourceSystemRepository;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

@ExtendWith(MockitoExtension.class)
class SourceSystemComponentTest {

@Mock
private WebClientProperties properties;
@Mock
private SourceSystemRepository repository;

@Test
void testGetSourceSystem() {
// Given
given(properties.getSourceSystemId()).willReturn(SOURCE_SYSTEM_ID);
given(repository.getSourceSystem(SOURCE_SYSTEM_ID)).willReturn(givenSourceSystemInformation());

// When
var component = new SourceSystemComponent(properties, repository);

// Then
assertThat(component.getSourceSystemName()).isEqualTo(SOURCE_SYSTEM_NAME);
assertThat(component.getSourceSystemID()).isEqualTo(SOURCE_SYSTEM_ID);
assertThat(component.getSourceSystemEndpoint()).isEqualTo(ENDPOINT);
}

@Test
void testFailedGetSourceSystem() {
// Given
given(properties.getSourceSystemId()).willReturn(SOURCE_SYSTEM_ID);
given(repository.getSourceSystem(SOURCE_SYSTEM_ID)).willReturn(null);

// When / Then
assertThrows(IllegalArgumentException.class,
() -> new SourceSystemComponent(properties, repository));
}
}
Loading

0 comments on commit 1815758

Please sign in to comment.