From 985b83790d8b2fc3b652a91d357e05a4b2a5aeef Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 4 Jul 2024 14:38:16 +0200 Subject: [PATCH] schema: make the schema compatible with 2.0 * application profile 2.0 changes some small things like contenthash -> identifier and fileurl to source to provide the naming for link only records. --- invenio_moodle/schemas.py | 38 +++++++++++++++++++++++++++----------- invenio_moodle/services.py | 11 +++++++---- invenio_moodle/utils.py | 31 ++++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/invenio_moodle/schemas.py b/invenio_moodle/schemas.py index 8307ccf..d6fc00a 100644 --- a/invenio_moodle/schemas.py +++ b/invenio_moodle/schemas.py @@ -13,6 +13,8 @@ from marshmallow import Schema, ValidationError, validates_schema from marshmallow.fields import Constant, Dict, List, Nested, Number, String +from .utils import extract_moodle_records + class ClassificationValuesSchema(Schema): """Moodle classification-values schema.""" @@ -79,12 +81,14 @@ class FileSchema(Schema): abstract = String(required=True) classification = List(Nested(ClassificationSchema), required=True) - contenthash = String(required=True) + contenthash = String() # application profile 1.0 + identifier = String() # application profile 2.0 context = String(required=True) courses = List(Nested(CourseSchema), required=True) filecreationtime = String(required=True) filesize = Number(required=True) - fileurl = String(required=True) + fileurl = String() # application profile 1.0 + source = String() # application profile 2.0 language = String(required=True) license = Nested(LicenseSchema) # noqa: A003 mimetype = String(required=True) @@ -101,6 +105,7 @@ class MoodleCourseSchema(Schema): """Moodle moodlecourse schema.""" files = List(Nested(FileSchema)) + elements = List(Nested(FileSchema)) class MoodleSchema(Schema): @@ -109,19 +114,12 @@ class MoodleSchema(Schema): Data coming from moodle should be in this format. """ - applicationprofile = String(required=True) - moodlecourses = Dict( - keys=String(), - values=Nested(MoodleCourseSchema, required=True), - ) - @validates_schema def validate_urls_unique(self, data: dict, **__: dict) -> None: """Check that each file-URL only appears once.""" urls_counter = Counter( - file_["fileurl"] - for _, moodlecourse in data["moodlecourses"].items() - for file_ in moodlecourse["files"] + file_["fileurl"] if "fileurl" in file_ else file_["source"] + for file_ in extract_moodle_records(data) ) duplicated_urls = [url for url, count in urls_counter.items() if count > 1] if duplicated_urls: @@ -157,3 +155,21 @@ def validate_course_jsons_unique_per_courseid(self, data: dict, **__: dict) -> N course_ids = ", ".join(course_id for course_id in ambiguous_courseids) msg = f"Different course-JSONs with same courseid {course_ids}." raise ValidationError(msg) + + +class MoodleSchemaApplicationProfile1(MoodleSchema): + """Moodle Schema for application profile 1.0.""" + + applicationprofile = String(required=True) + moodlecourses = Dict( + keys=String(), + values=Nested(MoodleCourseSchema, required=True), + ) + + +class MoodleSchemaApplicationProfile2(MoodleSchema): + """Moodle schema for application profile 2.0.""" + + applicationprofile = String(required=True) + release = Number() + elements = List(Nested(FileSchema)) diff --git a/invenio_moodle/services.py b/invenio_moodle/services.py index a568b5c..261a2c1 100644 --- a/invenio_moodle/services.py +++ b/invenio_moodle/services.py @@ -14,7 +14,7 @@ from marshmallow import ValidationError from .records import MoodleAPI, MoodleRESTConfig -from .schemas import MoodleSchema +from .schemas import MoodleSchemaApplicationProfile1, MoodleSchemaApplicationProfile2 from .types import FileCacheInfo from .utils import extract_moodle_records, post_processing @@ -47,9 +47,12 @@ def fetch_records(self, _: Identity) -> list[dict]: moodle_data = self.api.fetch_records() try: - MoodleSchema().load(moodle_data) - except ValidationError as error: - raise RuntimeError(str(error)) from error + MoodleSchemaApplicationProfile1().load(moodle_data) + except ValidationError: + try: + MoodleSchemaApplicationProfile2().load(moodle_data) + except ValidationError as error: + raise RuntimeError(str(error)) from error moodle_records = extract_moodle_records(moodle_data) post_processing(moodle_records) diff --git a/invenio_moodle/utils.py b/invenio_moodle/utils.py index 95f2c39..761cf30 100644 --- a/invenio_moodle/utils.py +++ b/invenio_moodle/utils.py @@ -20,11 +20,32 @@ def is_course_root(sourceid: str) -> bool: def extract_moodle_records(moodle_data: dict) -> list[dict]: """Create moodle file jsons.""" - return [ - file_json - for _, moodle_course in moodle_data["moodlecourses"].items() - for file_json in moodle_course["files"] - ] + + # application profile 2.0 uses elements and a flat structure to serve the metadata. + # { + # "elements": [ + # {METADATA goes here} + # ] + # } + + if "elements" in moodle_data: + return moodle_data["elements"] + + # application profile 1.0 uses a nested structure with + # { + # "moodlecourses":{ + # "1": { + # "files OR elements":[ + # {METADATA goes here} + # ] + # } + elements = [] + for _, moodle_course in moodle_data["moodlecourses"].items(): + if "files" in moodle_course: + elements.extend(file_json for file_json in moodle_course["files"]) + if "elements" in moodle_course: + elements.extend(file_json for file_json in moodle_course["elements"]) + return elements def remove_moodle_only_course(moodle_records: dict) -> None: