From 636f760a4576614b601890583d455ff53840c7b8 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Fri, 17 Dec 2021 10:15:17 -0500 Subject: [PATCH] wip: stateless pls_format_metadata api --- api/pls_format_metadata/urls.py | 8 +++++ api/pls_format_metadata/views.py | 43 +++++++++++++++++++++++ api/urls.py | 2 ++ share/metadata_formats/base.py | 9 ++++- share/metadata_formats/oai_dc.py | 3 ++ share/metadata_formats/sharev2_elastic.py | 4 +++ share/transform/base.py | 2 +- share/transform/chain/links.py | 2 +- share/transformers/oai.py | 1 - tests/api/test_pls_format_metadata.py | 26 ++++++++++++++ 10 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 api/pls_format_metadata/urls.py create mode 100644 api/pls_format_metadata/views.py create mode 100644 tests/api/test_pls_format_metadata.py diff --git a/api/pls_format_metadata/urls.py b/api/pls_format_metadata/urls.py new file mode 100644 index 000000000..bc8083c63 --- /dev/null +++ b/api/pls_format_metadata/urls.py @@ -0,0 +1,8 @@ +from django.urls import path + +from . import views + + +urlpatterns = [ + path('pls-format-metadata', views.pls_format_metadata), +] diff --git a/api/pls_format_metadata/views.py b/api/pls_format_metadata/views.py new file mode 100644 index 000000000..ba8c98000 --- /dev/null +++ b/api/pls_format_metadata/views.py @@ -0,0 +1,43 @@ +import json + +from django.http import HttpResponse + +from share.util.extensions import Extensions +from share.regulate import Regulator + + +def pls_format_metadata(request): + if request.method != 'POST': + return HttpResponse( + status=405, + headers={'Allow': 'POST'}, + content='only POST!' + ) + + transformer_key = request.GET.get('transformer', 'v2_push') + normal_graph = pls_normalize(request.body, transformer_key) + requested_formats = request.GET.getlist('formats') + formatted_records = [ + pls_format(normal_graph, format_key) + for format_key in requested_formats + ] + return HttpResponse( + content=json.dumps(formatted_records), + ) + + +def pls_normalize(raw_datum, transformer_key): + transformer = Extensions.get('share.transformers', transformer_key)() + graph = transformer.transform(raw_datum) + Regulator().regulate(graph) # in-place + + +def pls_format(graph, record_formats): + formatters = [ + Extensions.get('share.metadata_formats', record_format)() + for record_format in record_formats + ] + return [ + formatter.format_from_graph(graph) + for formatter in formatters + ] diff --git a/api/urls.py b/api/urls.py index bc41ad8b9..b1dbf214e 100644 --- a/api/urls.py +++ b/api/urls.py @@ -19,6 +19,8 @@ url('^', include('api.suids.urls')), url('^', include('api.users.urls')), + url('^', include('api.pls_format_metadata.urls')), + url('^schemas?/', include('api.schemas.urls'), name='schema'), url('^search/', include('api.search.urls'), name='search'), diff --git a/share/metadata_formats/base.py b/share/metadata_formats/base.py index 0714f46ac..0a3050372 100644 --- a/share/metadata_formats/base.py +++ b/share/metadata_formats/base.py @@ -3,11 +3,18 @@ from share.models.core import NormalizedData from share.models.ingest import SourceUniqueIdentifier +from share.util.graph import MutableGraph class MetadataFormatter(ABC): - @abstractmethod def format(self, normalized_data: NormalizedData) -> Optional[str]: + """return a string representation of the given metadata in the formatter's format + """ + mgraph = MutableGraph.from_jsonld(normalized_data.data) + return self.format_from_graph(mgraph) + + @abstractmethod + def format_from_graph(self, normalized_data: NormalizedData) -> Optional[str]: """return a string representation of the given metadata in the formatter's format """ raise NotImplementedError diff --git a/share/metadata_formats/oai_dc.py b/share/metadata_formats/oai_dc.py index a2edf71d3..891943e83 100644 --- a/share/metadata_formats/oai_dc.py +++ b/share/metadata_formats/oai_dc.py @@ -16,6 +16,9 @@ class OaiDcFormatter(MetadataFormatter): def format(self, normalized_datum): mgraph = MutableGraph.from_jsonld(normalized_datum.data) + return self.format_from_graph(mgraph) + + def format_from_graph(self, mgraph): central_work = mgraph.get_central_node(guess=True) if ( diff --git a/share/metadata_formats/sharev2_elastic.py b/share/metadata_formats/sharev2_elastic.py index 030ab611b..4c98f2476 100644 --- a/share/metadata_formats/sharev2_elastic.py +++ b/share/metadata_formats/sharev2_elastic.py @@ -58,6 +58,10 @@ def format_as_deleted(self, suid): 'is_deleted': True, }) + def format_from_graph(self, mgraph): + # HACK because sharev2_elastic is dumb + raise NotImplementedError('sharev2_elastic formatter depends on a NormalizedData') + def format(self, normalized_datum): mgraph = MutableGraph.from_jsonld(normalized_datum.data) central_work = mgraph.get_central_node(guess=True) diff --git a/share/transform/base.py b/share/transform/base.py index b8bf94888..ccbd5da61 100644 --- a/share/transform/base.py +++ b/share/transform/base.py @@ -6,7 +6,7 @@ class BaseTransformer(metaclass=abc.ABCMeta): - def __init__(self, source_config): + def __init__(self, source_config=None): self.config = source_config @abc.abstractmethod diff --git a/share/transform/chain/links.py b/share/transform/chain/links.py index 040f8d4f1..4a6bf3627 100644 --- a/share/transform/chain/links.py +++ b/share/transform/chain/links.py @@ -1029,7 +1029,7 @@ def execute(self, obj): break if not final[0]: - if self._urn_fallback: + if self._urn_fallback and getattr(Context(), '_config', None): urn = self.FALLBACK_FORMAT.format(source=Context()._config.label, id=urllib.parse.quote(obj)) return URNLink().execute(urn) else: diff --git a/share/transformers/oai.py b/share/transformers/oai.py index d8a47db0f..3bfd213c3 100644 --- a/share/transformers/oai.py +++ b/share/transformers/oai.py @@ -310,7 +310,6 @@ class RootParser(OAICreativeWork): type_map = root_type_map if property_list: - logger.debug('Attaching addition properties %s to transformer for %s', property_list, self.config.label) for prop in property_list: if prop in RootParser._extra: logger.warning('Skipping property %s, it already exists', prop) diff --git a/tests/api/test_pls_format_metadata.py b/tests/api/test_pls_format_metadata.py new file mode 100644 index 000000000..06e0d9129 --- /dev/null +++ b/tests/api/test_pls_format_metadata.py @@ -0,0 +1,26 @@ +from tests.share.metadata_formats.base import FORMATTER_TEST_INPUTS +from tests.share.metadata_formats.test_oai_dc_formatter import TestOaiDcFormatter as oaidc_test_cases + + +class TestPlsFormatMetadata: + def _get_test_keys(self): + return FORMATTER_TEST_INPUTS.keys() + + def _get_input(self, test_key): + return FORMATTER_TEST_INPUTS[test_key]['normalized_datum_kwargs']['data'] + + def _get_expected_output(self, test_key): + return oaidc_test_cases.expected_outputs[test_key] + + def test_works(self, client): + for test_key in self._get_test_keys(): + try: + resp = client.post( + '/api/v2/pls-format-metadata', + self._get_input(test_key), + ) + assert resp.status_code == 200 + assert resp.json() == self._get_expected_output(test_key) + print(f'success! ({test_key})') + except Exception as e: + print(f'fail! ({test_key}, {e})')