Merge pull request #261 from folio-org/MODOAIPMH-420

MODOAIPMH-420 - Save UUIDs of failed instances
folio-org · May 30, 2022 · c58d780 · c58d780
1 parent 133fac2
commit c58d780
Show file tree

Hide file tree

Showing 17 changed files with 553 additions and 62 deletions.
diff --git a/README.md b/README.md
@@ -80,6 +80,22 @@ At the first time, the last updated date is set during the initial-load and then
 
 But in a case when harvester lost his resumptionToken then saved instances ids with metadata that have not been retrieved and processed yet will be kept in DB and never will be cleaned. For preventing this case, request id has an expired period which for now equals to one day(24 h.) Therefore, when some request ids with expired last updated date exist then both such request ids and associated with them instances ids start to be considered as expired and will be removed from DB by cleaning job which are run each 2 hours.<br/>
 
+### Harvesting Statistics API
+
+Statistics API contains information on marc21_withholdings harvesting.
+The following endpoints can be used to monitor status of the completed harvesting.
+
+Name | Endpoint | Description 
+------------ | ------------- | ------------- 
+Request Metadata Collection | GET /oai/request-metadata | Returns whole collection of executed harvesting metadata 
+Failed to save instances UUIDs | GET /oai/request-metadata/{requestId}/failed-to-save-instances | Returns UUIDs collection of failed to save instances
+Failed instances UUIDs | GET /oai/request-metadata/{requestId}/failed-instances | Returns UUIDs collection of failed instances (instances were downloaded but failed to convert in MARC format)
+Skipped instances UUIDs| GET /oai/request-metadata/{requestId}/skipped-instances | Returns UUIDs collection of skipped instances (instances were downloaded but there are no corresponding records in SRS)
+Suppressed from discovery instances UUIDs| GET /oai/request-metadata/{requestId}/suppressed-from-discovery-instances | Returns UUIDs collection of suppressed from discovery instances
+
+A typical API usage should be performed with the following approach. The user requests a collection of Request Metadata. Finds the necessary request metadata by the harvesting start time. Request Metadata contains the `requestId` 
+and counters of the corresponding events. Next, the user can call the necessary endpoints using `requestId` to get a list of UUIDs.
+
 ### Issue tracker
 
 See project [MODOAIPMH](https://issues.folio.org/browse/MODOAIPMH)

diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json
@@ -4,7 +4,7 @@
   "provides": [
     {
       "id": "oai-pmh",
-      "version": "3.2",
+      "version": "3.3",
       "handlers": [
         {
           "methods": ["GET"],
@@ -63,6 +63,26 @@
           "pathPattern": "/oai/request-metadata",
           "permissionsRequired": ["oai-pmh.request-metadata.collection.get"]
         },
+        {
+          "methods": ["GET"],
+          "pathPattern": "/oai/request-metadata/{requestId}/failed-to-save-instances",
+          "permissionsRequired": ["oai-pmh.request-metadata.failed-to-save-instances.collection.get"]
+        },
+        {
+          "methods": ["GET"],
+          "pathPattern": "/oai/request-metadata/{requestId}/failed-instances",
+          "permissionsRequired": ["oai-pmh.request-metadata.failed-instances.collection.get"]
+        },
+        {
+          "methods": ["GET"],
+          "pathPattern": "/oai/request-metadata/{requestId}/skipped-instances",
+          "permissionsRequired": ["oai-pmh.request-metadata.skipped-instances.collection.get"]
+        },
+        {
+          "methods": ["GET"],
+          "pathPattern": "/oai/request-metadata/{requestId}/suppressed-from-discovery-instances",
+          "permissionsRequired": ["oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get"]
+        },
         {
           "methods": [
             "POST"
@@ -142,6 +162,26 @@
       "displayName": "OAI-PMH Request metadata - get collection",
       "description": "Retrieves request metadata collection"
     },
+    {
+      "permissionName": "oai-pmh.request-metadata.failed-to-save-instances.collection.get",
+      "displayName": "OAI-PMH Request metadata - get collection of failed to save instances UUIDs",
+      "description": "Retrieves request metadata collection - failed to save instances UUIDs"
+    },
+    {
+      "permissionName": "oai-pmh.request-metadata.failed-instances.collection.get",
+      "displayName": "OAI-PMH Request metadata - get collection of failed instances UUIDs",
+      "description": "Retrieves request metadata collection - failed instances UUIDs"
+    },
+    {
+      "permissionName": "oai-pmh.request-metadata.skipped-instances.collection.get",
+      "displayName": "OAI-PMH Request metadata - get collection of skipped instances UUIDs",
+      "description": "Retrieves request metadata collection - skipped instances UUIDs"
+    },
+    {
+      "permissionName": "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get",
+      "displayName": "OAI-PMH Request metadata - get collection of suppressed from discovery instances UUIDs",
+      "description": "Retrieves request metadata collection - suppressed from discovery instances UUIDs"
+    },
     {
       "permissionName": "oai-pmh.all",
       "displayName": "OAI-PMH - all permissions",
@@ -154,7 +194,11 @@
         "oai-pmh.sets.item.delete",
         "oai-pmh.sets.item.collection.get",
         "oai-pmh.filtering-conditions.get",
-        "oai-pmh.request-metadata.collection.get"
+        "oai-pmh.request-metadata.collection.get",
+        "oai-pmh.request-metadata.failed-to-save-instances.collection.get",
+        "oai-pmh.request-metadata.failed-instances.collection.get",
+        "oai-pmh.request-metadata.skipped-instances.collection.get",
+        "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get"
       ]
     },
     {

diff --git a/ramls/examples/requestMetadata.sample b/ramls/examples/requestMetadata.sample
@@ -2,8 +2,10 @@
     "requestId" : "a8e1d986-2eef-4007-827d-59c49f640086",
     "lastUpdatedDate" : "2022-04-28T10:56:15.192+00:00",
     "streamEnded" : true,
+    "downloadedAndSavedInstancesCounter" : 20,
+    "failedToSaveInstancesCounter" : 10,
     "returnedInstancesCounter" : 1,
     "skippedInstancesCounter" : 0,
     "failedInstancesCounter" : 1,
-    "supressedInstancesCounter" : 0
-  }
+    "suppressedInstancesCounter" : 0
+}
diff --git a/ramls/examples/requestMetadataCollection.sample b/ramls/examples/requestMetadataCollection.sample
@@ -3,10 +3,12 @@
     "requestId" : "a8e1d986-2eef-4007-827d-59c49f640086",
     "lastUpdatedDate" : "2022-04-28T10:56:15.192+00:00",
     "streamEnded" : true,
+    "downloadedAndSavedInstancesCounter" : 20,
+    "failedToSaveInstancesCounter" : 10,
     "returnedInstancesCounter" : 1,
     "skippedInstancesCounter" : 0,
     "failedInstancesCounter" : 1,
-    "supressedInstancesCounter" : 0
+    "suppressedInstancesCounter" : 0
   } ],
   "totalRecords" : 1
 }
diff --git a/ramls/examples/uuidCollection.sample b/ramls/examples/uuidCollection.sample
@@ -0,0 +1,7 @@
+{
+  "uuidCollection" : [
+    "a8e1d986-2eef-4007-827d-59c49f640086",
+    "ca78250e-16a3-4719-93ec-b506934f40be"
+  ],
+  "totalRecords" : 2
+}
diff --git a/ramls/request-metadata.raml b/ramls/request-metadata.raml
@@ -10,7 +10,12 @@ documentation:
 types:
   request_metadata: !include schemas/dto/requestMetadata.json
   request_metadata_collection: !include schemas/dto/requestMetadataCollection.json
+  uuid: !include schemas/common/uuid.json
+  uuid_collection: !include schemas/dto/uuidCollection.json
   errors: !include raml-util/schemas/errors.schema
+  UUID:
+    type: string
+    pattern: ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$
 
 traits:
   pageable:  !include raml-util/traits/pageable.raml
@@ -19,8 +24,11 @@ traits:
 
 resourceTypes:
   collection-get: !include raml-util/rtypes/collection-get.raml
+  get-only: !include raml-util/rtypes/get-only.raml
 
 /oai/request-metadata:
+  displayName: OAIPMH Request Metadata API
+  description: Service that allows to retrieve Request Metadata Collection
   type:
     collection-get:
       exampleCollection: !include examples/requestMetadata.sample
@@ -32,3 +40,46 @@ resourceTypes:
     description: Get list of request metadata
     is: [pageable]
 
+  /{requestId}/failed-to-save-instances:
+    description: Service that allows to retrieve UUIDs of failed to save instances
+    type:
+      get-only:
+        exampleCollection: !include examples/uuidCollection.sample
+        schema: uuid_collection
+    is: [ validate ]
+    get:
+      description: Get list of failed to save instances UUIDs
+      is: [ pageable ]
+
+  /{requestId}/skipped-instances:
+    description: Service that allows to retrieve UUIDs of skipped instances
+    type:
+      get-only:
+        exampleCollection: !include examples/uuidCollection.sample
+        schema: uuid_collection
+    is: [ validate ]
+    get:
+      description: Get list of skipped instances UUIDs
+      is: [ pageable ]
+
+  /{requestId}/failed-instances:
+    description: Service that allows to retrieve UUIDs of failed instances
+    type:
+      get-only:
+        exampleCollection: !include examples/uuidCollection.sample
+        schema: uuid_collection
+    is: [ validate ]
+    get:
+      description: Get list of failed instances UUIDs
+      is: [ pageable ]
+
+  /{requestId}/suppressed-from-discovery-instances:
+    description: Service that allows to retrieve UUIDs of suppressed from discovery instances
+    type:
+      get-only:
+        exampleCollection: !include examples/uuidCollection.sample
+        schema: uuid_collection
+    is: [ validate ]
+    get:
+      description: Get list of suppressed from discovery instances UUIDs
+      is: [ pageable ]
diff --git a/ramls/schemas/dto/requestMetadata.json b/ramls/schemas/dto/requestMetadata.json
@@ -38,8 +38,8 @@
       "description": "failed instances counter",
       "type": "integer"
     },
-    "supressedInstancesCounter": {
-      "description": "supressed instances counter",
+    "suppressedInstancesCounter": {
+      "description": "suppressed instances counter",
       "type": "integer"
     }
   }

diff --git a/ramls/schemas/dto/uuidCollection.json b/ramls/schemas/dto/uuidCollection.json
@@ -0,0 +1,25 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "UUID collection",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "uuidCollection" : {
+      "description": "list of UUIDs",
+      "type": "array",
+      "id": "uuidCollection",
+      "items": {
+        "type": "object",
+        "$ref": "../common/uuid.json"
+      }
+    },
+    "totalRecords": {
+      "type": "integer",
+      "description": "Total number of records"
+    }
+  },
+  "required": [
+    "uuidCollection",
+    "totalRecords"
+  ]
+}
diff --git a/src/main/java/org/folio/oaipmh/dao/InstancesDao.java b/src/main/java/org/folio/oaipmh/dao/InstancesDao.java
@@ -5,6 +5,7 @@
 
 import org.folio.oaipmh.domain.StatisticsHolder;
 import org.folio.rest.jaxrs.model.RequestMetadataCollection;
+import org.folio.rest.jaxrs.model.UuidCollection;
 import org.folio.rest.jooq.tables.pojos.Instances;
 import org.folio.rest.jooq.tables.pojos.RequestMetadataLb;
 
@@ -24,6 +25,14 @@ public interface InstancesDao {
    */
   Future<RequestMetadataCollection> getRequestMetadataCollection(int offset, int limit, String tenantId);
 
+  Future<UuidCollection> getFailedToSaveInstancesIdsCollection(String requestId, int offset, int limit, String tenantId);
+
+  Future<UuidCollection> getSkippedInstancesIdsCollection(String requestId, int offset, int limit, String tenantId);
+
+  Future<UuidCollection> getFailedInstancesIdsCollection(String requestId, int offset, int limit, String tenantId);
+
+  Future<UuidCollection> getSuppressedInstancesIdsCollection(String requestId, int offset, int limit, String tenantId);
+
   /**
    * Saves specified request metadata. Entity must contain request id, in opposite case IllegalStateException will be thrown.
    */