documents: optimize availability

- Uses Elasticsearch to compute the documents, items and holdings availability.
- Search only one organisation when an organisation is retrived using a view code.
- Renames the available property for items and holdings into a `is_available` method.

Co-Authored-by: Johnny Mariéthoz <[email protected]>

pyproject.toml

@@ -165,7 +165,6 @@ documents = "rero_ils.modules.documents.api_views:api_blueprint"
 circ_policies = "rero_ils.modules.circ_policies.views:blueprint"
 local_entities = "rero_ils.modules.entities.local_entities.views:api_blueprint"
 remote_entities = "rero_ils.modules.entities.remote_entities.views:api_blueprint"
-documents = "rero_ils.modules.documents.views:api_blueprint"
 holdings = "rero_ils.modules.holdings.api_views:api_blueprint"
 imports = "rero_ils.modules.imports.views:api_blueprint"
 item_types = "rero_ils.modules.item_types.views:blueprint"

rero_ils/modules/documents/api.py

@@ -161,27 +161,125 @@ def _validate(self, **kwargs):
         return json
 
     @classmethod
-    def is_available(cls, pid, view_code, raise_exception=False):
-        """Get availability for document."""
-        from ..holdings.api import Holding
+    def get_n_available_holdings(cls, pid, org_pid=None):
+        """Get the number of available and electronic holdings.
+
+        :param pid: str - the document pid.
+        :param org_pid: str - the organisation pid.
+        :returns: int, int - the number of available and electronic holdings.
+        """
+        from rero_ils.modules.holdings.api import HoldingsSearch
+
+        # create the holding search query
+        holding_query = HoldingsSearch().available_query()
+
+        # filter by the current document
+        filters = Q('term', document__pid=pid)
+
+        # filter by organisation
+        if org_pid:
+            filters &= Q('term', organisation__pid=org_pid)
+        holding_query = holding_query.filter(filters)
+
+        # get the number of electronic holdings
+        n_electronic_holdings = holding_query\
+            .filter('term', holdings_type='electronic')
+
+        return holding_query.count(), n_electronic_holdings.count()
+
+    @classmethod
+    def get_available_item_pids(cls, pid, org_pid=None):
+        """Get the list of the available item pids.
+
+        :param pid: str - the document pid.
+        :param org_pid: str - the organisation pid.
+        :returns: [str] - the list of the available item pids.
+        """
+        from rero_ils.modules.items.api import ItemsSearch
+
+        # create the item query
+        items_query = ItemsSearch().available_query()
+
+        # filter by the current document
+        filters = Q('term', document__pid=pid)
+
+        # filter by organisation
+        if org_pid:
+            filters &= Q('term', organisation__pid=org_pid)
+
+        return [
+            hit.pid for hit in items_query.filter(filters).source('pid').scan()
+        ]
+
+    @classmethod
+    def get_item_pids_with_active_loan(cls, pid, org_pid=None):
+        """Get the list of items pids that have active loans.
+
+        :param pid: str - the document pid.
+        :param org_pid: str - the organisation pid.
+        :returns: [str] - the list of the item pids having active loans.
+        """
+        from rero_ils.modules.loans.api import LoansSearch
+
+        loan_query = LoansSearch().unavailable_query()
+
+        # filter by the current document
+        filters = Q('term', document_pid=pid)
+
+        # filter by organisation
+        if org_pid:
+            filters &= Q('term', organisation__pid=org_pid)
+
+        loan_query = loan_query.filter(filters)
+
+        return [
+            hit.item_pid.value for hit in loan_query.source('item_pid').scan()
+        ]
+
+    @classmethod
+    def is_available(cls, pid, view_code=None):
+        """Get availability for document.
+
+        Note: if the logic has to be changed here please check also for items
+        and holdings availability.
+
+        :param pid: str - document pid value.
+        :param view_code: str - the view code.
+        """
+        # get the organisation pid corresponding to the view code
+        org_pid = None
         if view_code != current_app.config.get(
                 'RERO_ILS_SEARCH_GLOBAL_VIEW_CODE'):
-            view_id = Organisation.get_record_by_viewcode(view_code)['pid']
-            holding_pids = Holding.get_holdings_pid_by_document_pid_by_org(
-                    pid, view_id)
-        else:
-            holding_pids = Holding.get_holdings_pid_by_document_pid(pid)
-        for holding_pid in holding_pids:
-            if holding := Holding.get_record_by_pid(holding_pid):
-                if holding.available:
-                    return True
-            else:
-                msg = f'No holding: {holding_pid} in DB ' \
-                      f'for document: {pid}'
-                current_app.logger.error(msg)
-                if raise_exception:
-                    raise ValueError(msg)
-        return False
+            org_pid = Organisation.get_record_by_viewcode(view_code)['pid']
+
+        # -------------- Holdings --------------------
+        # get the number of available and electronic holdings
+        n_available_holdings, n_electronic_holdings = \
+            cls.get_n_available_holdings(pid, org_pid)
+
+        # available if an electronic holding exists
+        if n_electronic_holdings:
+            return True
+
+        # unavailable if no holdings exists
+        if not n_available_holdings:
+            return False
+
+        # -------------- Items --------------------
+        # get the available item pids
+        available_item_pids = cls.get_available_item_pids(pid, org_pid)
+
+        # unavailable if no items exists
+        if not available_item_pids:
+            return False
+
+        # --------------- Loans -------------------
+        # get item pids that have active loans
+        unavailable_item_pids = \
+            cls.get_item_pids_with_active_loan(pid, org_pid)
+
+        # available if at least one item don't have active loan
+        return bool(set(available_item_pids) - set(unavailable_item_pids))
 
     @property
     def harvested(self):

rero_ils/modules/documents/views.py

@@ -24,9 +24,7 @@
 
 import click
 from elasticsearch_dsl.query import Q
-from flask import Blueprint, current_app, render_template
-from flask import url_for
-from flask import Blueprint, current_app, render_template
+from flask import Blueprint, current_app, render_template, url_for
 from flask_babelex import gettext as _
 from flask_login import current_user
 from invenio_records_ui.signals import record_viewed
@@ -41,7 +39,7 @@
 from rero_ils.modules.locations.api import Location
 from rero_ils.modules.organisations.api import Organisation
 from rero_ils.modules.patrons.api import current_patrons
-from rero_ils.modules.utils import cached, extracted_data_from_ref
+from rero_ils.modules.utils import extracted_data_from_ref
 
 from .api import Document, DocumentsSearch
 from .extensions import EditionStatementExtension, \

rero_ils/modules/holdings/api.py

@@ -84,6 +84,14 @@ class Meta:
 
         default_filter = None
 
+    def available_query(self):
+        """Base query for holding availability.
+
+        :returns: a filtered Elasticsearch query.
+        """
+        # should not masked
+        return self.exclude('term', _masked=True)
+
 
 class Holding(IlsRecord):
     """Holding class."""
@@ -260,14 +268,65 @@ def vendor(self):
         if self.get('vendor'):
             return extracted_data_from_ref(self.get('vendor'), data='record')
 
-    @property
-    def available(self):
-        """Get availability for holding."""
+    def get_available_item_pids(self):
+        """Get the list of the available item pids.
+
+        :returns: [str] - the list of the available item pids.
+        """
+        from rero_ils.modules.items.api import ItemsSearch
+        items_query = ItemsSearch().available_query()
+        filters = Q('term', holding__pid=self.pid)
+        return [
+            hit.pid for hit in items_query.filter(filters).source('pid').scan()
+        ]
+
+    def get_item_pids_with_active_loan(self, item_pids):
+        """Get the list of items pids that have active loans.
+
+        :param item_pids: [str] - the list of the item pids.
+        :returns: the list of the item pids having active loans.
+        """
+        from rero_ils.modules.loans.api import LoansSearch
+
+        loan_query = LoansSearch().unavailable_query()
+
+        # the loans corresponding to the given item pids
+        loan_query = loan_query.filter(Q('terms', item_pid__value=item_pids))
+
+        return [
+            hit.item_pid.value for hit in loan_query.source('item_pid').scan()
+        ]
+
+    def is_available(self):
+        """Get availability for the current holding.
+
+        Note: if the logic has to be changed here please check also for items
+        and documents availability.
+        """
+        # -------------- Holdings --------------------
+        # unavailable if masked
+        if self.get('_masked', False):
+            return False
+
+        # available if the holding is electronic
         if self.is_electronic:
             return True
-        if self.get('_masked', False):
+
+        # -------------- Items --------------------
+        # get available item pids
+        available_item_pids = self.get_available_item_pids()
+
+        # unavailable if no item exists
+        if not available_item_pids:
             return False
-        return self._exists_available_child()
+
+        # --------------- Loans -------------------
+        # get item pids that have active loans
+        unavailable_item_pids = \
+            self.get_item_pids_with_active_loan(available_item_pids)
+
+        # available if at least one item don't have active loan
+        return bool(set(available_item_pids) - set(unavailable_item_pids))
 
     @property
     def max_number_of_claims(self):
@@ -303,14 +362,6 @@ def get_note(self, note_type):
                  if note.get('type') == note_type]
         return next(iter(notes), None)
 
-    def _exists_available_child(self):
-        """Check if at least one child of this holding is available."""
-        for pid in Item.get_items_pid_by_holding_pid(self.pid):
-            item = Item.get_record_by_pid(pid)
-            if item.available:
-                return True
-        return False
-
     @property
     def get_items_count_by_holding_pid(self):
         """Returns items count from holding pid."""

rero_ils/modules/holdings/api_views.py

@@ -315,6 +315,6 @@ def holding_availability(pid):
     """HTTP GET request for holding availability."""
     if holding := Holding.get_record_by_pid(pid):
         return jsonify({
-            'available': holding.available
+            'available': holding.is_available()
         })
     abort(404)

rero_ils/modules/items/api/api.py

@@ -21,12 +21,14 @@
 from functools import partial
 
 from elasticsearch.exceptions import NotFoundError
+from elasticsearch_dsl import Q
 from invenio_search import current_search_client
 
 from rero_ils.modules.api import IlsRecordError, IlsRecordsIndexer, \
     IlsRecordsSearch
 from rero_ils.modules.documents.api import DocumentsSearch
 from rero_ils.modules.fetchers import id_fetcher
+from rero_ils.modules.item_types.api import ItemTypesSearch
 from rero_ils.modules.minters import id_minter
 from rero_ils.modules.organisations.api import Organisation
 from rero_ils.modules.patrons.api import current_librarian
@@ -62,6 +64,37 @@ class Meta:
 
         default_filter = None
 
+    def available_query(self):
+        """Base elasticsearch query to compute availability.
+
+        :returns: elasticsearch query.
+        """
+        must_not_filters = [
+            # should not be masked
+            Q('term', _masked=True),
+            # if issue the status should be received
+            Q('exists', field='issue') & ~Q('term', issue__status='received')
+        ]
+
+        # negative availability item types
+        not_available_item_types = [
+            hit.pid for hit in ItemTypesSearch()
+            .source('pid')
+            .filter('term', negative_availability=True)
+            .scan()
+        ]
+        if not_available_item_types:
+            # negative availability item type and not temporary item types
+            has_items_filters = \
+                Q('terms', item_type__pid=not_available_item_types)
+            has_items_filters &= ~Q('exists', field='temporary_item_type')
+            # temporary item types with negative availability
+            has_items_filters |= Q(
+                'terms', temporary_item_type__pid=not_available_item_types)
+            # add to the must not filters
+            must_not_filters.append(has_items_filters)
+        return self.filter(Q('bool', must_not=must_not_filters))
+
 
 class Item(ItemCirculation, ItemIssue):
     """Item class."""

rero_ils/modules/items/api/circulation.py

@@ -1276,26 +1276,23 @@ def get_loan_states_for_an_item(self):
             ]).params(preserve_order=True).source(['state'])
         return list(dict.fromkeys([result.state for result in search.scan()]))
 
-    @property
-    def available(self):
+    def is_available(self):
         """Get availability for item.
 
-        An item is 'available' if there are no related request/active_loan and
-        if the related circulation category doesn't specify a negative
-        availability.
-        All masked items are considered as unavailable.
+        Note: if the logic has to be changed here please check also for
+        documents and holdings availability.
         """
-        if self.get('_masked', False):
-            return False
-        if self.circulation_category.get('negative_availability'):
-            return False
-        if self.temp_item_type_negative_availability:
-            return False
-        if self.is_issue and self.issue_status != ItemIssueStatus.RECEIVED:
-            return False
-        if self.item_has_active_loan_or_request() > 0:
+        from ..api import ItemsSearch
+
+        items_query = ItemsSearch().available_query()
+
+        # check item availability
+        if not items_query.filter('term', pid=self.pid).count():
             return False
-        return True
+
+        # --------------- Loans -------------------
+        # unavailable if the current item has active loans
+        return not self.item_has_active_loan_or_request()
 
     @property
     def availability_text(self):

rero_ils/modules/items/utils.py

@@ -89,7 +89,7 @@ def exists_available_item(items=None):
             item = Item.get_record_by_pid(item)
         if not isinstance(item, Item):
             raise ValueError('All items should be Item resource.')
-        if item.available:
+        if item.is_available():
             return True
     return False
 

rero_ils/modules/items/views/api_views.py

@@ -472,7 +472,7 @@ def item_availability(pid):
     item = Item.get_record_by_pid(pid)
     if not item:
         abort(404)
-    data = dict(available=item.available)
+    data = dict(available=item.is_available())
     if flask_request.args.get('more_info'):
         extra = {
             'status': item['status'],