diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json
index ffae1b7a4..63b157728 100644
--- a/descriptors/ModuleDescriptor-template.json
+++ b/descriptors/ModuleDescriptor-template.json
@@ -203,7 +203,8 @@
"inventory-storage.instance-types.collection.get",
"inventory-storage.nature-of-content-terms.item.get",
"inventory-storage.instance-formats.item.get",
- "inventory-storage.instance-note-types.item.get"
+ "inventory-storage.instance-note-types.item.get",
+ "source-storage.sourceRecords.get"
]
},
{
diff --git a/folio-export-common b/folio-export-common
index a31fe18b8..7439ca7f0 160000
--- a/folio-export-common
+++ b/folio-export-common
@@ -1 +1 @@
-Subproject commit a31fe18b8a62d16ea73fd37f766afa2e35865cba
+Subproject commit 7439ca7f0ba5976eacde0f101052497f0a4ba7c6
diff --git a/pom.xml b/pom.xml
index 6efdec822..4cacd6a3a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -59,6 +59,7 @@
3.7.3
5.7.1
12.1
+ 2.9.2
2.4.0
@@ -240,6 +241,12 @@
3.1.0-M1
+
+ org.marc4j
+ marc4j
+ ${marc4j.version}
+
+
diff --git a/src/main/java/org/folio/dew/batch/MarcAsListStringsWriter.java b/src/main/java/org/folio/dew/batch/MarcAsListStringsWriter.java
index f9438a6bb..d17f4d45d 100644
--- a/src/main/java/org/folio/dew/batch/MarcAsListStringsWriter.java
+++ b/src/main/java/org/folio/dew/batch/MarcAsListStringsWriter.java
@@ -3,6 +3,8 @@
import lombok.extern.log4j.Log4j2;
import org.folio.dew.client.SrsClient;
import org.folio.dew.domain.dto.Formatable;
+import org.folio.dew.error.BulkEditException;
+import org.folio.dew.service.JsonToMarcConverter;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.ExecutionContext;
@@ -10,10 +12,14 @@
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.util.Assert;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import static java.lang.String.format;
import static java.util.Objects.nonNull;
+import static org.folio.dew.utils.Constants.NO_MARC_CONTENT;
@Log4j2
@StepScope
@@ -21,17 +27,27 @@ public class MarcAsListStringsWriter> extends FlatFil
private SrsClient srsClient;
private MarcAsStringWriter delegateToStringWriter;
+ private JsonToMarcConverter jsonToMarcConverter;
- public MarcAsListStringsWriter(String outputFileName, SrsClient srsClient) {
+ public MarcAsListStringsWriter(String outputFileName, SrsClient srsClient, JsonToMarcConverter jsonToMarcConverter) {
super();
this.srsClient = srsClient;
+ this.jsonToMarcConverter = jsonToMarcConverter;
delegateToStringWriter = new MarcAsStringWriter<>(outputFileName);
}
@Override
public void write(Chunk> items) throws Exception {
- delegateToStringWriter.write(new Chunk<>(items.getItems().stream().flatMap(List::stream).filter(itm -> itm.isInstanceFormat() && itm.isSourceMarc()).map(marc -> getMarcContent(marc.getId()))
- .filter(Objects::nonNull).toList()));
+ delegateToStringWriter.write(new Chunk<>(items.getItems().stream().flatMap(List::stream)
+ .filter(itm -> itm.isInstanceFormat() && itm.isSourceMarc()).map(marc -> {
+ try {
+ return getMarcContent(marc.getId());
+ } catch (Exception e) {
+ log.error(e);
+ throw new BulkEditException(format(NO_MARC_CONTENT, marc.getId(), e.getMessage()));
+ }
+ })
+ .flatMap(List::stream).filter(Objects::nonNull).toList()));
}
@Override
@@ -60,15 +76,19 @@ public void close() {
}
}
- private String getMarcContent(String id) {
- var srsRecords = srsClient.getMarc(id, "INSTANCE");
- if (srsRecords.getSourceRecords().isEmpty()) {
+ private List getMarcContent(String id) throws Exception {
+ List mrcRecords = new ArrayList<>();
+ var srsRecords = srsClient.getMarc(id, "INSTANCE").get("sourceRecords");
+ if (srsRecords.isEmpty()) {
log.warn("No SRS records found by instanceId = {}", id);
- return null;
+ return mrcRecords;
}
- var recordId = srsRecords.getSourceRecords().get(0).getRecordId();
- var marcRecord = srsClient.getMarcContent(recordId);
- log.info("MARC record found by recordId = {}", recordId);
- return marcRecord.getRawRecord().getContent();
+ for (var jsonNodeIterator = srsRecords.elements(); jsonNodeIterator.hasNext();) {
+ var srsRec = jsonNodeIterator.next();
+ var parsedRec = srsRec.get("parsedRecord");
+ var content = parsedRec.get("content").toString();
+ mrcRecords.add(jsonToMarcConverter.convertJsonRecordToMarcRecord(content));
+ }
+ return mrcRecords;
}
}
diff --git a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditInstanceIdentifiersJobConfig.java b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditInstanceIdentifiersJobConfig.java
index 136c52da6..986cd5890 100644
--- a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditInstanceIdentifiersJobConfig.java
+++ b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditInstanceIdentifiersJobConfig.java
@@ -13,6 +13,7 @@
import org.folio.dew.domain.dto.ItemIdentifier;
import org.folio.dew.error.BulkEditException;
import org.folio.dew.error.BulkEditSkipListener;
+import org.folio.dew.service.JsonToMarcConverter;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.StepScope;
@@ -43,6 +44,7 @@ public class BulkEditInstanceIdentifiersJobConfig {
private final BulkEditInstanceProcessor bulkEditInstanceProcessor;
private final BulkEditSkipListener bulkEditSkipListener;
private final SrsClient srsClient;
+ private final JsonToMarcConverter jsonToMarcConverter;
@Bean
public Job bulkEditProcessInstanceIdentifiersJob(JobCompletionNotificationListener listener, Step bulkEditInstanceStep,
@@ -80,7 +82,7 @@ public CompositeItemWriter> compositeInstanceListWriter(@Va
@Value("#{jobParameters['" + TEMP_LOCAL_MARC_PATH + "']}") String outputMarcName) {
var writer = new CompositeItemWriter>();
writer.setDelegates(Arrays.asList(new CsvListFileWriter<>(outputFileName, InstanceFormat.getInstanceColumnHeaders(), InstanceFormat.getInstanceFieldsArray(), (field, i) -> field),
- new JsonListFileWriter<>(new FileSystemResource(outputFileName + ".json")), new MarcAsListStringsWriter<>(outputMarcName, srsClient)));
+ new JsonListFileWriter<>(new FileSystemResource(outputFileName + ".json")), new MarcAsListStringsWriter<>(outputMarcName, srsClient, jsonToMarcConverter)));
return writer;
}
}
diff --git a/src/main/java/org/folio/dew/client/SrsClient.java b/src/main/java/org/folio/dew/client/SrsClient.java
index a2cc2f7e7..af54d8347 100644
--- a/src/main/java/org/folio/dew/client/SrsClient.java
+++ b/src/main/java/org/folio/dew/client/SrsClient.java
@@ -1,19 +1,15 @@
package org.folio.dew.client;
+import com.fasterxml.jackson.databind.JsonNode;
import org.folio.dew.config.feign.FeignClientConfiguration;
-import org.folio.dew.domain.dto.MarcRecord;
-import org.folio.dew.domain.dto.SrsRecordCollection;
import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
@FeignClient(name = "source-storage", configuration = FeignClientConfiguration.class)
public interface SrsClient {
- @GetMapping(value = "/source-records")
- SrsRecordCollection getMarc(@RequestParam("instanceId") String instanceId, @RequestParam("idType") String idType);
-
- @GetMapping(value = "/records/{srsId}")
- MarcRecord getMarcContent(@PathVariable String srsId);
+ @GetMapping(value = "/source-records", produces = MediaType.APPLICATION_JSON_VALUE)
+ JsonNode getMarc(@RequestParam("instanceId") String instanceId, @RequestParam("idType") String idType);
}
diff --git a/src/main/java/org/folio/dew/service/JsonToMarcConverter.java b/src/main/java/org/folio/dew/service/JsonToMarcConverter.java
new file mode 100644
index 000000000..05d92fef1
--- /dev/null
+++ b/src/main/java/org/folio/dew/service/JsonToMarcConverter.java
@@ -0,0 +1,43 @@
+package org.folio.dew.service;
+
+import lombok.extern.log4j.Log4j2;
+import org.marc4j.MarcException;
+import org.marc4j.MarcJsonReader;
+import org.marc4j.MarcStreamWriter;
+import org.springframework.stereotype.Component;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+@Log4j2
+@Component
+public class JsonToMarcConverter {
+
+ public String convertJsonRecordToMarcRecord(String jsonRecord) throws IOException {
+ var byteArrayInputStream = new ByteArrayInputStream(jsonRecord.getBytes(StandardCharsets.UTF_8));
+ var byteArrayOutputStream = new ByteArrayOutputStream();
+ try (byteArrayInputStream; byteArrayOutputStream) {
+ var marcJsonReader = new MarcJsonReader(byteArrayInputStream);
+ var marcStreamWriter = new MarcStreamWriter(byteArrayOutputStream, StandardCharsets.UTF_8.name());
+ writeMarc(marcJsonReader, marcStreamWriter);
+ return byteArrayOutputStream.toString();
+ } catch (IOException e) {
+ log.error(e.getMessage());
+ throw e;
+ }
+ }
+
+ private void writeMarc(MarcJsonReader marcJsonReader, MarcStreamWriter marcStreamWriter) {
+ try {
+ while (marcJsonReader.hasNext()) {
+ var marc = marcJsonReader.next();
+ marcStreamWriter.write(marc);
+ }
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ throw new MarcException(e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/org/folio/dew/utils/Constants.java b/src/main/java/org/folio/dew/utils/Constants.java
index 3b2badf90..34ad3e9c2 100644
--- a/src/main/java/org/folio/dew/utils/Constants.java
+++ b/src/main/java/org/folio/dew/utils/Constants.java
@@ -52,6 +52,7 @@ public class Constants {
public static final String STATUS_FIELD_CAN_NOT_CLEARED = "Status field can not be cleared";
public static final String STATUS_VALUE_NOT_ALLOWED = "New status value \"%s\" is not allowed";
public static final String MULTIPLE_MATCHES_MESSAGE = "Multiple matches for the same identifier.";
+ public static final String NO_MARC_CONTENT = "Cannot get marc content for record with id = %s, reason: %s";
public static final String MODULE_NAME = "BULKEDIT";
public static final String BULKEDIT_DIR_NAME = "bulk_edit";
diff --git a/src/main/resources/swagger.api/bulk-edit.yaml b/src/main/resources/swagger.api/bulk-edit.yaml
index aaad1c060..f295d6e52 100644
--- a/src/main/resources/swagger.api/bulk-edit.yaml
+++ b/src/main/resources/swagger.api/bulk-edit.yaml
@@ -683,14 +683,6 @@ components:
$ref: '../../../../folio-export-common/schemas/inventory/identifierTypeReferenceCollection.json#/IdentifierTypeReferenceCollection'
InstanceNoteType:
$ref: '../../../../folio-export-common/schemas/inventory/instanceNoteType.json#/InstanceNoteType'
- RawRecord:
- $ref: '../../../../folio-export-common/schemas/srs/rawRecord.json#/RawRecord'
- MarcRecord:
- $ref: '../../../../folio-export-common/schemas/srs/marcRecord.json#/MarcRecord'
- SrsRecord:
- $ref: '../../../../folio-export-common/schemas/srs/srsRecord.json#/SrsRecord'
- SrsRecordCollection:
- $ref: '../../../../folio-export-common/schemas/srs/srsRecordCollection.json#/SrsRecordCollection'
examples:
errors:
value:
diff --git a/src/test/java/org/folio/dew/BulkEditTest.java b/src/test/java/org/folio/dew/BulkEditTest.java
index fe361d8e9..a3dc05264 100644
--- a/src/test/java/org/folio/dew/BulkEditTest.java
+++ b/src/test/java/org/folio/dew/BulkEditTest.java
@@ -109,6 +109,7 @@ class BulkEditTest extends BaseBatchTest {
private static final String ITEM_BARCODES_CSV = "src/test/resources/upload/item_barcodes.csv";
private static final String INSTANCE_HRIDS_CSV = "src/test/resources/upload/instance_hrids.csv";
private static final String MARC_INSTANCE_ID_CSV = "src/test/resources/upload/marc_instance_id.csv";
+ private static final String MARC_INSTANCE_ID_INVALID_CONTENT_CSV = "src/test/resources/upload/marc_instance_id_invalid_content.csv";
private static final String MARC_INSTANCE_HRID_CSV = "src/test/resources/upload/marc_instance_hrid.csv";
private static final String INSTANCE_ISSN_ISBN_CSV = "src/test/resources/upload/instance_ISSN_ISBN.csv";
private static final String ITEM_BARCODES_DOUBLE_QOUTES_CSV = "src/test/resources/upload/item_barcodes_double_qoutes.csv";
@@ -324,7 +325,58 @@ void uploadMarcInstanceIdentifiersJobTest(String identifierType, String path) th
final FileSystemResource actualResult = actualFileOutput(jobExecution.getExecutionContext().getString(OUTPUT_FILES_IN_STORAGE).split(";")[3]);
- assertEquals("marc content", new String(actualResult.getContentAsByteArray()));
+ assertEquals("00026nam a2200025 a 4500\u001E\u001D", new String(actualResult.getContentAsByteArray()));
+ assertThat(jobExecution.getExitStatus()).isEqualTo(ExitStatus.COMPLETED);
+ }
+
+ @Test
+ void uploadMarcInstanceIdentifiersInvalidContentJobTest() throws Exception {
+
+ var path = MARC_INSTANCE_ID_INVALID_CONTENT_CSV;
+ JobLauncherTestUtils testLauncher = createTestLauncher(bulkEditProcessInstanceIdentifiersJob);
+
+ var parametersBuilder = new JobParametersBuilder();
+ String jobId = UUID.randomUUID().toString();
+ String workDir = getWorkingDirectory(springApplicationName, BULKEDIT_DIR_NAME);
+ parametersBuilder.addString(TEMP_OUTPUT_MARC_PATH, workDir + jobId + "/" + "marc_instance_id");
+ parametersBuilder.addString(TEMP_LOCAL_MARC_PATH,
+ getTempDirWithSeparatorSuffix() + springApplicationName + PATH_SEPARATOR + jobId + PATH_SEPARATOR + "marc_instance_id");
+ parametersBuilder.addString(TEMP_LOCAL_FILE_PATH,
+ getTempDirWithSeparatorSuffix() + springApplicationName + PATH_SEPARATOR + jobId + PATH_SEPARATOR + "out");
+ parametersBuilder.addString(TEMP_OUTPUT_FILE_PATH, workDir + jobId + "/" + "out");
+ try {
+ localFilesStorage.write(workDir + "marc_instance_id", new byte[32]);
+ localFilesStorage.write(workDir+ jobId + "/marc_instance_id.mrc", new byte[32]);
+ localFilesStorage.write(workDir + "out", new byte[0]);
+ localFilesStorage.write(workDir + "out.csv", new byte[0]);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ Path of = Path.of(path);
+ var file = getWorkingDirectory("mod-data-export-worker", BULKEDIT_DIR_NAME) +
+ FilenameUtils.removeExtension((new File(path)).getName()) + "E" + FilenameUtils.getExtension(path);
+ parametersBuilder.addString(FILE_NAME, file);
+ localFilesStorage.write(file, Files.readAllBytes(of));
+ parametersBuilder.addLong(TOTAL_CSV_LINES, countLines(localFilesStorage, file, false), false);
+
+ var tempDir = getTempDirWithSeparatorSuffix() + springApplicationName + PATH_SEPARATOR + jobId;
+ var tempFile = tempDir + PATH_SEPARATOR + of.getFileName();
+ Files.createDirectories(Path.of(tempDir));
+ Files.write(Path.of(tempFile), Files.readAllBytes(of));
+ parametersBuilder.addString(TEMP_IDENTIFIERS_FILE_NAME, tempFile);
+
+ parametersBuilder.addString(JobParameterNames.JOB_ID, jobId);
+ parametersBuilder.addString(EXPORT_TYPE, BULK_EDIT_IDENTIFIERS.getValue());
+ parametersBuilder.addString(ENTITY_TYPE, INSTANCE.getValue());
+ parametersBuilder.addString(IDENTIFIER_TYPE, "ID");
+
+ final JobParameters jobParameters = parametersBuilder.toJobParameters();
+
+ JobExecution jobExecution = testLauncher.launchJob(jobParameters);
+
+ final FileSystemResource actualResult = actualFileOutput(jobExecution.getExecutionContext().getString(OUTPUT_FILES_IN_STORAGE).split(";")[3]);
+
+ assertEquals("", new String(actualResult.getContentAsByteArray()).trim());
assertThat(jobExecution.getExitStatus()).isEqualTo(ExitStatus.COMPLETED);
}
diff --git a/src/test/resources/mappings/instances-query.json b/src/test/resources/mappings/instances-query.json
index e05a63d5e..990dc524f 100644
--- a/src/test/resources/mappings/instances-query.json
+++ b/src/test/resources/mappings/instances-query.json
@@ -221,6 +221,19 @@
}
}
},
+ {
+ "request": {
+ "method": "GET",
+ "url": "/inventory/instances?query=id%3D%3D7772796a-b88b-4991-a9f7-2e368217c487&limit=1"
+ },
+ "response": {
+ "status": 200,
+ "body": "{\n \"instances\": [\n {\n \"id\": \"7772796a-b88b-4991-a9f7-2e368217c487\",\n \"_version\": \"3\",\n \"hrid\": \"inst000000000022\",\n \"source\": \"MARC\",\n \"title\": \"American Bar Association journal.\",\n \"administrativeNotes\": [],\n \"indexTitle\": \"American Bar Association journal.\",\n \"parentInstances\": [],\n \"childInstances\": [],\n \"isBoundWith\": false,\n \"alternativeTitles\": [],\n \"editions\": [],\n \"series\": [],\n \"identifiers\": [],\n \"contributors\": [\n {\n \"authorityId\": null,\n \"contributorNameTypeId\": \"d376e36c-b759-4fed-8502-7130d1eeff39\",\n \"name\": \"American Bar Association\",\n \"contributorTypeId\": \"6e09d47d-95e2-4d8a-831b-f777b8ef6d81\",\n \"contributorTypeText\": \"\",\n \"primary\": null\n },\n {\n \"authorityId\": null,\n \"contributorNameTypeId\": \"d376e36c-b759-4fed-8502-7130d1eeff39\",\n \"name\": \"American Bar Association. Journal\",\n \"contributorTypeId\": \"06b2cbd8-66bf-4956-9d90-97c9776365a4\",\n \"contributorTypeText\": \"\",\n \"primary\": null\n }\n ],\n \"subjects\": [],\n \"classifications\": [],\n \"publication\": [],\n \"publicationFrequency\": [\n \"Monthly, 1921-83\",\n \"Quarterly, 1915-20\"\n ],\n \"publicationRange\": [\n \"Began with vol. 1, no. 1 (Jan. 1915); ceased with v. 69, [no.12] (Dec. 1983)\"\n ],\n \"electronicAccess\": [],\n \"instanceTypeId\": \"30fffe0e-e985-4144-b2e2-1e8179bdb41f\",\n \"instanceFormatIds\": [\n \"5cb91d15-96b1-4b8a-bf60-ec310538da66\"\n ],\n \"physicalDescriptions\": [\n \"69 v. : ill. ; 23-30 cm.\"\n ],\n \"languages\": [\n \"eng\"\n ],\n \"notes\": [],\n \"previouslyHeld\": false,\n \"discoverySuppress\": false,\n \"statisticalCodeIds\": [],\n \"metadata\": {\n \"createdDate\": \"2023-11-10T11:54:16.187+00:00\",\n \"createdByUserId\": \"cffb2565-07fc-470b-86c6-17d8ce14432e\",\n \"updatedDate\": \"2024-01-05T09:52:26.651+00:00\",\n \"updatedByUserId\": \"ca6022de-2644-46fe-b6f2-78df15483721\"\n },\n \"tags\": {\n \"tagList\": []\n },\n \"natureOfContentTermIds\": [\n\t\t\t\t\"921e6d93-bafb-4a02-b62f-dcd027c45406\"\n\t\t\t],\n \"precedingTitles\": [],\n \"succeedingTitles\": []\n }\n ],\n \"totalRecords\": 1\n}",
+ "headers": {
+ "Content-Type": "application/json"
+ }
+ }
+ },
{
"request": {
"method": "GET",
diff --git a/src/test/resources/mappings/srs-record-invalid-content.json b/src/test/resources/mappings/srs-record-invalid-content.json
new file mode 100644
index 000000000..53d41aa15
--- /dev/null
+++ b/src/test/resources/mappings/srs-record-invalid-content.json
@@ -0,0 +1,17 @@
+{
+ "mappings": [
+ {
+ "request": {
+ "method": "GET",
+ "url": "/source-storage/source-records?instanceId=7772796a-b88b-4991-a9f7-2e368217c487&idType=INSTANCE"
+ },
+ "response": {
+ "status": 200,
+ "body": "{\n \"sourceRecords\": [\n {\n \"recordId\": \"777fad9e-7f8e-4d8e-9a71-00d251817866\", \"parsedRecord\": {\n \"id\": \"8e5ea07d-0c06-4a3b-ab34-f4fc3f76bc09\",\n \"conten\": {\"ghf\": \"marc content\"} }}\n ],\n \"totalRecords\": 1\n}",
+ "headers": {
+ "Content-Type": "application/json"
+ }
+ }
+ }
+ ]
+}
diff --git a/src/test/resources/mappings/srs-records.json b/src/test/resources/mappings/srs-records.json
index 1ec17317f..1c8c96768 100644
--- a/src/test/resources/mappings/srs-records.json
+++ b/src/test/resources/mappings/srs-records.json
@@ -7,7 +7,7 @@
},
"response": {
"status": 200,
- "body": "{\n \"sourceRecords\": [\n {\n \"recordId\": \"666fad9e-7f8e-4d8e-9a71-00d251817866\" }\n ],\n \"totalRecords\": 1\n}",
+ "body": "{\n \"sourceRecords\": [\n {\n \"recordId\": \"666fad9e-7f8e-4d8e-9a71-00d251817866\", \"parsedRecord\": {\n \"id\": \"8e5ea07d-0c06-4a3b-ab34-f4fc3f76bc09\",\n \"content\": {\"000\": \"marc content\"} }\n }\n ],\n \"totalRecords\": 1\n}",
"headers": {
"Content-Type": "application/json"
}
diff --git a/src/test/resources/upload/marc_instance_id_invalid_content.csv b/src/test/resources/upload/marc_instance_id_invalid_content.csv
new file mode 100644
index 000000000..e9dceccc2
--- /dev/null
+++ b/src/test/resources/upload/marc_instance_id_invalid_content.csv
@@ -0,0 +1 @@
+7772796a-b88b-4991-a9f7-2e368217c487