From ff0ebe6f5a3a77cefc4b620eff769e6eb4c427bb Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Fri, 13 Dec 2024 12:31:34 +0100 Subject: [PATCH] [app][feat] enable MODS 3.8 validation --- pyproject.toml | 2 +- src/digiflow/resources/xsd/mods-3-8.xsd | 1770 ++++++++++++++++++++ src/digiflow/validate/metadata_xsd.py | 64 +- tests/resources/1877049026_Aa_mods38.xml | 1925 ++++++++++++++++++++++ tests/test_validate_metadata.py | 2 +- 5 files changed, 3729 insertions(+), 34 deletions(-) create mode 100644 src/digiflow/resources/xsd/mods-3-8.xsd create mode 100755 tests/resources/1877049026_Aa_mods38.xml diff --git a/pyproject.toml b/pyproject.toml index 1efff4c..e9bddcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "digiflow" -version = "5.6.9" +version = "5.7.9" description = "Father's Little Digitization Workflow Helper" readme = "README.md" requires-python = ">=3.8" diff --git a/src/digiflow/resources/xsd/mods-3-8.xsd b/src/digiflow/resources/xsd/mods-3-8.xsd new file mode 100644 index 0000000..a31587a --- /dev/null +++ b/src/digiflow/resources/xsd/mods-3-8.xsd @@ -0,0 +1,1770 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/digiflow/validate/metadata_xsd.py b/src/digiflow/validate/metadata_xsd.py index 0784578..9d22e29 100644 --- a/src/digiflow/validate/metadata_xsd.py +++ b/src/digiflow/validate/metadata_xsd.py @@ -9,31 +9,32 @@ _XDS_RES = pathlib.Path(__file__).parent.parent / 'resources' / 'xsd' - - -METS_1_12 = os.path.join(_XDS_RES, 'mets_1-12.xsd') -MODS_3_7 = os.path.join(_XDS_RES, 'mods_3-7.xsd') -MIX_2_0 = os.path.join(_XDS_RES, 'mix_2-0.xsd') -ALTO_3_1 = os.path.join(_XDS_RES, 'alto_3-1.xsd') -ALTO_4_2 = os.path.join(_XDS_RES, 'alto_4-2.xsd') +METS_1_12 = os.path.join(_XDS_RES, 'mets-1-12.xsd') +MODS_3_8 = os.path.join(_XDS_RES, 'mods-3-8.xsd') +MODS_3_7 = os.path.join(_XDS_RES, 'mods-3-7.xsd') +MIX_2_0 = os.path.join(_XDS_RES, 'mix-2-0.xsd') +ALTO_3_1 = os.path.join(_XDS_RES, 'alto-3-1.xsd') +ALTO_4_2 = os.path.join(_XDS_RES, 'alto-4-2.xsd') METS_MODS_XSD = {'mets:mets': [METS_1_12], - 'mods:mods': [MODS_3_7]} -_DEFAULT_XSD_MAPPINGS = {'mets:mets': [METS_1_12], 'mods:mods': [MODS_3_7], 'mix:mix': [ - MIX_2_0], 'alto': [ALTO_4_2]} + 'mods:mods': [MODS_3_8]} +DEFAULT_XSD_MAPPINGS = { + 'mets:mets': [METS_1_12], + 'mods:mods': [MODS_3_8], #MODS_3_7], + 'mix:mix': [MIX_2_0], + 'alto': [ALTO_4_2], +} # please linter for lxml # pylint: disable=c-extension-no-member - class InvalidXMLException(Exception): """Mark invalid Validation outcome""" -def __is_schema_root(xml_tree, schema) -> bool: +def _is_schema_root(xml_tree, schema) -> bool: """ - Schema *might* be prefixed, like mets:mets, *or* not like ALTO-files - Therefore we go for QName + Rooot namespace *might* be prefixed, therefore go for tree's QName """ qualified_name = ET.QName(xml_tree) local_name = qualified_name.localname @@ -41,40 +42,39 @@ def __is_schema_root(xml_tree, schema) -> bool: def _is_contained(xml_tree, schema): - if __is_schema_root(xml_tree, schema): + if _is_schema_root(xml_tree, schema): return True return len(xml_tree.findall('.//' + schema, dfc.XMLNS)) > 0 def _validate(xml_tree, schema, xsd_file): - if __is_schema_root(xml_tree, schema): + if _is_schema_root(xml_tree, schema): return _validate_with_xsd(xml_tree, xsd_file) - else: - _invalids = [] - sections = xml_tree.findall('.//' + schema, dfc.XMLNS) - for section in sections: - _invalids.extend(_validate_with_xsd(section, xsd_file)) - return _invalids + _invalids = [] + sections = xml_tree.findall('.//' + schema, dfc.XMLNS) + for section in sections: + _invalids.extend(_validate_with_xsd(section, xsd_file)) + return _invalids def _validate_with_xsd(xml_tree, xsd_file): the_tree = ET.parse(xsd_file) schema_tree = ET.XMLSchema(the_tree) - _invalids = [] + invalids = [] try: schema_tree.assertValid(xml_tree) except ET.DocumentInvalid: for error in schema_tree.error_log: - _entry = (error.level_name, error.domain_name, error.message) - _invalids.append(_entry) - return _invalids + entry = (error.level_name, error.domain_name, error.message) + invalids.append(entry) + return invalids def validate_xml(xml_data, xsd_mappings=None) -> bool: """ Validate XML data with a set of given schema definitions (XSDs) - :param xml_data: string|PosixPath to file or ET.etree.root + :param xml_data: string|PosixPath|ET.etree.root """ if isinstance(xml_data, pathlib.Path): @@ -82,12 +82,12 @@ def validate_xml(xml_data, xsd_mappings=None) -> bool: if isinstance(xml_data, str): xml_data = ET.parse(xml_data).getroot() if xsd_mappings is None: - xsd_mappings = _DEFAULT_XSD_MAPPINGS - _invalids = [] + xsd_mappings = DEFAULT_XSD_MAPPINGS + invalids = [] for schema, xsd_files in xsd_mappings.items(): if _is_contained(xml_data, schema): for xsd_file in xsd_files: - _invalids.extend(_validate(xml_data, schema, xsd_file)) - if len(_invalids) > 0: - raise InvalidXMLException(_invalids) + invalids.extend(_validate(xml_data, schema, xsd_file)) + if len(invalids) > 0: + raise InvalidXMLException(invalids) return True diff --git a/tests/resources/1877049026_Aa_mods38.xml b/tests/resources/1877049026_Aa_mods38.xml new file mode 100755 index 0000000..4468ca0 --- /dev/null +++ b/tests/resources/1877049026_Aa_mods38.xml @@ -0,0 +1,1925 @@ + + + + 118918 + + + + + + Text + + 1685 + [1685?] + + Hall in Sachsen + + Zufinden bey Simon Johann Hübnern + [1685?] + + urn:nbn:de:gbv:3:1-ha32-171133730-1877049026-10 + vd17 3310:757822R + |fei|ger- ens- n.it al16 3 + + 1877049026 + rda + 02-01-24 + 24-06-24 + + + ger + + + 8 ungezählte Seiten, 134 Seiten, 2 ungezählte Seiten + + Verordnung + + + Kurfürstl. Herzogtum publiziertes Grafschaft Mansfeld Donnerstags + Vierwöchentliche Bettag Bußtag beigefügten Bußpsalmen sonderbare + + Aa + + Text + txt + + 1516514412012/8792 + + Zufinden bey Simon Johann Hübnern + Zufinden bey Simon Johann Hübnern + + + + + Marienbibliothek Halle + Oelh 21 Dd (1) + + 090 + Public + Domain Mark 1.0 + + + + + + + + + + [I.] + + + + + + + + + + + + + + [II.] + + + + + + + + + + + + + + Allgemeines Kirchen-Gebet. + + + + + + + + + + + + + + [Psalmen] + + + + + + + + + + + + + + Vermahnung zur Busse. + + + + + + + + + + + + + + Gebet In Serbensläuften. + + + + + + + + + + + + + + Tägliche Todes-Bereitung. + + + + + + + + + + + + + + Register/ Der Gesänge so in diesen Büchlein zufinden. + + + + + + + + + + + + + + Universitäts- und Landesbibliothek Sachsen-Anhalt + Share_it + http://www.bibliothek.uni-halle.de + mailto:auskunft@bibliothek.uni-halle.de + pdm + + + + + + + + + Share_it + http://opac.bibliothek.uni-halle.de/DB=1/CLK?IKT=12&TRM=1877049026 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_validate_metadata.py b/tests/test_validate_metadata.py index a5c4bb5..9cd6ffd 100644 --- a/tests/test_validate_metadata.py +++ b/tests/test_validate_metadata.py @@ -142,7 +142,7 @@ def test_most_rectent_mods38(tmp_path): data includes brand new mods:displayDate""" # arrange - path_src = os.path.join(TEST_RES, "1877049026_Aa_valid.xml") + path_src = os.path.join(TEST_RES, "1877049026_Aa_mods38.xml") tmp_dst = tmp_path / '1877049026.xml' shutil.copyfile(path_src, tmp_dst)