api: fix sources filter

* Corrects source filter.
* Better GND get online record for redirected records.
* Dependencies updates.

Co-Authored-by: Peter Weber <[email protected]>

rero_mef/agents/api.py

@@ -155,7 +155,7 @@ def create_or_update_mef(self, dbcommit=False, reindex=False,
         return mef_record, mef_actions
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online Record.
 
         Has to be overloaded in agent class.

rero_mef/agents/gnd/api.py

@@ -53,15 +53,15 @@ class AgentGndRecord(AgentRecord):
     search = AgentGndSearch
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online record.
 
         :param id: Id of online record.
         :param verbose: Verbosity.
         :returns: record or None
         """
         from .tasks import gnd_get_record
-        return gnd_get_record(id=id, debug=debug)
+        return gnd_get_record(id_=id_, debug=debug)
 
 
 class AgentGndIndexer(AgentIndexer):

rero_mef/agents/gnd/tasks.py

@@ -90,7 +90,7 @@ def save_records_from_dates(file_name, from_date=None, until_date=None,
 
 
 @shared_task
-def gnd_get_record(id, debug=False):
+def gnd_get_record(id_, debug=False):
     """Get a record from GND SRU repo.
 
     GND documentation:
@@ -99,16 +99,21 @@ def gnd_get_record(id, debug=False):
     &operation=searchRetrieve&query=idn%3D007355440&recordSchema=MARC21-xml
     """
     url = current_app.config.get(
-        'RERO_MEF_AGENTS_GND_GET_RECORD').replace('{id}', id)
+        'RERO_MEF_AGENTS_GND_GET_RECORD').replace('{id}', id_)
     trans_record = None
-    msg = f'SRU-agents.gnd  get: {id:<15} {url}'
+    msg = f'SRU-agents.gnd  get: {id_:<15} {url}'
     try:
         response = requests_retry_session().get(url)
         status_code = response.status_code
         if status_code == requests.codes.ok:
             if records := parse_xml_to_array(BytesIO(response.content)):
                 trans_record = Transformation(records[0]).json
-                msg = f'{msg} | OK'
+                pid = trans_record.get('pid')
+                if id_ != trans_record.get('pid'):
+                    msg = f'{msg} | PID changed: {id_} -> {pid}'
+                    trans_record = None
+                else:
+                    msg = f'{msg} | OK'
             else:
                 msg = f'{msg} | No record'
         else:

rero_mef/agents/idref/api.py

@@ -53,10 +53,10 @@ class AgentIdrefRecord(AgentRecord):
     search = AgentIdrefSearch
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online record."""
         from .tasks import idref_get_record
-        return idref_get_record(id=id, debug=debug)
+        return idref_get_record(id_=id_, debug=debug)
 
 
 class AgentIdrefIndexer(AgentIndexer):

rero_mef/agents/idref/tasks.py

@@ -86,10 +86,10 @@ def save_records_from_dates(file_name, from_date=None, until_date=None,
 
 
 @shared_task
-def idref_get_record(id, verbose=False, debug=False):
+def idref_get_record(id_, verbose=False, debug=False):
     """Get a record from IDREF OAI repo."""
     return oai_get_record(
-        id=id,
+        id_=id_,
         name='agents.idref',
         transformation=Transformation,
         identifier='oai:IdRefOAIServer.fr:',

rero_mef/agents/rero/api.py

@@ -53,10 +53,10 @@ class AgentReroRecord(AgentRecord):
     search = AgentReroSearch
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online record."""
         from .tasks import rero_get_record
-        return rero_get_record(id=id, debug=debug)
+        return rero_get_record(id_=id_, debug=debug)
 
 
 class AgentReroIndexer(AgentIndexer):

rero_mef/agents/rero/tasks.py

@@ -28,15 +28,15 @@
 
 
 @shared_task
-def rero_get_record(id, debug=False):
+def rero_get_record(id_, debug=False):
     """Get a record from RERO data repo.
 
     RERO documentation:
     http://data.rero.ch/
     http://data.rero.ch/02-A000069866/marcxml
     """
     url = current_app.config.get(
-        'RERO_MEF_AGENTS_RERO_GET_RECORD').replace('{id}', id)
+        'RERO_MEF_AGENTS_RERO_GET_RECORD').replace('{id}', id_)
     msg = f'API-agents.rero  get: {id:<15} {url}'
     trans_record = None
     try:

rero_mef/agents/viaf/api.py

@@ -207,7 +207,7 @@ def update_online(agent_class, pid, online):
             action = Action.NOT_ONLINE
             agent_record = None
             if agent_class.provider.pid_type in online:
-                data, msg = agent_class.get_online_record(id=pid)
+                data, msg = agent_class.get_online_record(id_=pid)
                 if online_verbose:
                     click.echo(msg)
                 if data and not data.get('NO TRANSFORMATION'):

rero_mef/concepts/api.py

@@ -93,7 +93,7 @@ def create_or_update_mef(self, dbcommit=False, reindex=False):
         return mef_record, {mef_record.pid: mef_action}
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online Record.
 
         Has to be overloaded in agent class.

rero_mef/concepts/idref/api.py

@@ -53,13 +53,13 @@ class ConceptIdrefRecord(ConceptRecord):
     search = ConceptIdrefSearch
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online Record.
 
         Has to be overloaded in agent class.
         """
         from .tasks import idref_get_record
-        return idref_get_record(id=id, debug=debug)
+        return idref_get_record(id_=id_, debug=debug)
 
 
 class ConceptIdrefIndexer(ConceptIndexer):

rero_mef/concepts/idref/tasks.py

@@ -86,10 +86,10 @@ def save_records_from_dates(file_name, from_date=None, until_date=None,
 
 
 @shared_task
-def idref_get_record(id, debug=False):
+def idref_get_record(id_, debug=False):
     """Get a record from IDREF OAI repo."""
     return oai_get_record(
-        id=id,
+        id_=id_,
         name='concepts.idref',
         transformation=Transformation,
         identifier='oai:IdRefOAIServer.fr:',

rero_mef/config.py

@@ -521,7 +521,7 @@
             type=dict(
                 terms=dict(field='type', size=30)
             ),
-            sources=dict(
+            source=dict(
                 terms=dict(field='sources', size=30)
             ),
             deleted=dict(
@@ -533,7 +533,7 @@
         ),
         filters=dict(
             type=terms_filter('type'),
-            sources=terms_filter('sources'),
+            source=terms_filter('sources'),
             deleted=exists_filter('deleted'),
             deleted_entities=exists_filter('*.deleted'),
             rero_double=terms_filter('rero.pid')
@@ -590,7 +590,7 @@
             type=dict(
                 terms=dict(field='type', size=30)
             ),
-            sources=dict(
+            source=dict(
                 terms=dict(field='sources', size=30)
             ),
             deleted=dict(
@@ -602,7 +602,7 @@
         ),
         filters=dict(
             type=terms_filter('type'),
-            sources=terms_filter('sources'),
+            source=terms_filter('sources'),
             deleted=exists_filter('deleted'),
             deleted_entities=exists_filter('*.deleted'),
             rero_double=terms_filter('rero.pid')
@@ -663,7 +663,7 @@
             type=dict(
                 terms=dict(field='type', size=30)
             ),
-            sources=dict(
+            source=dict(
                 terms=dict(field='sources', size=30)
             ),
             deleted=dict(
@@ -675,7 +675,7 @@
         ),
         filters=dict(
             type=terms_filter('type'),
-            sources=terms_filter('sources'),
+            source=terms_filter('sources'),
             deleted=exists_filter('deleted'),
             deleted_entities=exists_filter('*.deleted')
         )

rero_mef/places/api.py

@@ -93,7 +93,7 @@ def create_or_update_mef(self, dbcommit=False, reindex=False):
         return mef_record, {mef_record.pid: mef_action}
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online Record.
 
         Has to be overloaded in agent class.

rero_mef/places/idref/api.py

@@ -53,10 +53,10 @@ class PlaceIdrefRecord(PlaceRecord):
     search = PlaceIdrefSearch
 
     @classmethod
-    def get_online_record(cls, id, debug=False):
+    def get_online_record(cls, id_, debug=False):
         """Get online Record."""
         from .tasks import idref_get_record
-        return idref_get_record(id=id, debug=debug)
+        return idref_get_record(id_=id_, debug=debug)
 
 
 class PlaceIdrefIndexer(PlaceIndexer):

rero_mef/places/idref/tasks.py

@@ -86,10 +86,10 @@ def save_records_from_dates(file_name, from_date=None, until_date=None,
 
 
 @shared_task
-def idref_get_record(id, debug=False):
+def idref_get_record(id_, debug=False):
     """Get a record from IDREF OAI repo."""
     return oai_get_record(
-        id=id,
+        id_=id_,
         name='places.idref',
         transformation=Transformation,
         identifier='oai:IdRefOAIServer.fr:',

rero_mef/utils.py

@@ -472,7 +472,7 @@ def oai_save_records_from_dates(name, file_name, sickle, oai_item_iterator,
     return count
 
 
-def oai_get_record(id, name, transformation, access_token=None,
+def oai_get_record(id_, name, transformation, access_token=None,
                    identifier=None, debug=False, **kwargs):
     """Get record from an OAI repo.
 
@@ -489,20 +489,20 @@ def oai_get_record(id, name, transformation, access_token=None,
 
     params = {
         'metadataPrefix': metadata_prefix,
-        'identifier': f'{identifier}{id}'
+        'identifier': f'{identifier}{id_}'
     }
     full_url = f'{url}?verb=GetRecord&metadataPrefix={metadata_prefix}'
-    full_url = f'{full_url}&identifier={identifier}{id}'
+    full_url = f'{full_url}&identifier={identifier}{id_}'
 
     if access_token:
         params['accessToken'] = access_token
         full_url = f'{full_url}&accessToken={access_token}'
 
     try:
         record = request.GetRecord(**params)
-        msg = f'OAI-{name:<12} get: {id:<15} {full_url} | OK'
+        msg = f'OAI-{name:<12} get: {id_:<15} {full_url} | OK'
     except Exception as err:
-        msg = f'OAI-{name:<12} get: {id:<15} {full_url} | NO RECORD'
+        msg = f'OAI-{name:<12} get: {id_:<15} {full_url} | NO RECORD'
         if debug:
             raise
         return None, msg

scripts/test

@@ -63,6 +63,8 @@ if [[ -z "${VIRTUAL_ENV}" ]]; then
 fi
 
 set -e
+# -> Vulnerability found in flask-caching version 2.0.1
+#    Vulnerability ID: 40459
 # -> Vulnerability found in flask-security version 3.0.0
 #    Vulnerability ID: 45183
 # -> Vulnerability found in flask-security version 3.0.0
@@ -75,7 +77,8 @@ set -e
 #    Vulnerability ID: 42852
 # -> Vulnerability found in py version 1.11.0
 #    Vulnerability ID: 51457
-safety check -o bare -i 45183 -i 44501 -i 51668 -i 42194 -i 42852 -i 51457
+info_msg "Test safety:"
+safety check -o bare -i 40459 -i 45183 -i 44501 -i 51668 -i 42194 -i 42852 -i 51457
 info_msg "Test pydocstyle:"
 pydocstyle rero_mef tests docs
 info_msg "Test isort:"

tests/api/test_agents_mef_rest.py

@@ -43,7 +43,7 @@ def test_view_agents_mef(client, agent_mef_record, agent_gnd_record,
             'doc_count_error_upper_bound': 0,
             'sum_other_doc_count': 0
         },
-        'sources': {
+        'source': {
             'buckets': [
                 {'doc_count': 1, 'key': 'gnd'},
                 {'doc_count': 1, 'key': 'idref'},

tests/api/test_concepts_mef_rest.py

@@ -40,7 +40,7 @@ def test_view_concepts_mef(client,
     assert json_data['aggregations'] == {
         'deleted': {'doc_count': 0},
         'deleted_entities': {'doc_count': 0},
-        'sources': {
+        'source': {
             'buckets': [{
                 'doc_count': 1,
                 'key': "idref"

tests/api/test_places_mef_rest.py

@@ -43,7 +43,7 @@ def test_view_places_mef(client, place_mef_idref_redirect_record,
     assert json_data['aggregations'] == {
         'deleted': {'doc_count': 0},
         'deleted_entities': {'doc_count': 0},
-        'sources': {
+        'source': {
             'buckets': [{
                 'doc_count': 1,
                 'key': "idref"