From 74527c17af35bd563b8e8513755093324745487a Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 12:48:03 +0200 Subject: [PATCH] feat: onthology validation --- raillabel_providerkit/exceptions.py | 6 + raillabel_providerkit/validation/__init__.py | 2 + .../validation/validate_onthology/__init__.py | 3 + .../_onthology_classes/__init__.py | 2 + .../_attributes/__init__.py | 2 + .../_attributes/_attribute_abc.py | 52 ++ .../_attributes/_boolean_attribute.py | 29 + .../_attributes/_integer_attribute.py | 29 + .../_attributes/_multi_select_attribute.py | 51 ++ .../_attributes/_single_select_attribute.py | 50 ++ .../_attributes/_string_attribute.py | 29 + .../_attributes/_vector_attribute.py | 29 + .../_onthology_classes/_object_classes.py | 113 +++ .../_onthology_classes/_onthology.py | 46 ++ .../_onthology_classes/_sensor_type.py | 34 + .../validate_onthology/validate_onthology.py | 66 ++ .../test_onthology_schema_v1.py | 3 +- .../test_validate_onthology.py | 705 ++++++++++++++++++ 18 files changed, 1250 insertions(+), 1 deletion(-) create mode 100644 raillabel_providerkit/validation/validate_onthology/__init__.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py create mode 100644 raillabel_providerkit/validation/validate_onthology/validate_onthology.py create mode 100644 tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py diff --git a/raillabel_providerkit/exceptions.py b/raillabel_providerkit/exceptions.py index 37319d8..47ff8f7 100644 --- a/raillabel_providerkit/exceptions.py +++ b/raillabel_providerkit/exceptions.py @@ -12,3 +12,9 @@ class SchemaError(Exception): """Raised when the data does not validate against a given schema.""" __module__ = "raillabel_providerkit" + + +class OnthologySchemaError(Exception): + """Raised when the .yaml-file provided is not valid against the schema.""" + + __module__ = "raillabel_providerkit" diff --git a/raillabel_providerkit/validation/__init__.py b/raillabel_providerkit/validation/__init__.py index 4c38e62..098b319 100644 --- a/raillabel_providerkit/validation/__init__.py +++ b/raillabel_providerkit/validation/__init__.py @@ -1,3 +1,5 @@ # Copyright DB Netz AG and contributors # SPDX-License-Identifier: Apache-2.0 """Package for validating raillabel data regarding the format requirements.""" + +from .validate_onthology.validate_onthology import validate_onthology diff --git a/raillabel_providerkit/validation/validate_onthology/__init__.py b/raillabel_providerkit/validation/validate_onthology/__init__.py new file mode 100644 index 0000000..56bbc60 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/__init__.py @@ -0,0 +1,3 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 +"""Package for validating a scene via an onthology.""" diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py new file mode 100644 index 0000000..aabc64e --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py @@ -0,0 +1,2 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py new file mode 100644 index 0000000..aabc64e --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py @@ -0,0 +1,2 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py new file mode 100644 index 0000000..0599d7b --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py @@ -0,0 +1,52 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import typing as t +from dataclasses import dataclass +from importlib import import_module +from inspect import isclass +from pathlib import Path +from pkgutil import iter_modules + + +@dataclass +class _Attribute(abc.ABC): + @abc.abstractclassmethod + def supports(cls, data_dict: dict) -> bool: + raise NotImplementedError + + @abc.abstractclassmethod + def fromdict(cls, data_dict: dict) -> t.Type["_Attribute"]: + raise NotImplementedError + + @abc.abstractmethod + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + raise NotImplementedError + + +def attribute_classes() -> list[t.Type[_Attribute]]: + """Return dictionary with Attribute child classes.""" + return ATTRIBUTE_CLASSES + + +def _collect_attribute_classes(): + """Collect attribute child classes and store them.""" + + global ATTRIBUTE_CLASSES + + package_dir = str(Path(__file__).resolve().parent) + for (_, module_name, _) in iter_modules([package_dir]): + + module = import_module( + f"raillabel_providerkit.validation.validate_onthology._onthology_classes._attributes.{module_name}" + ) + for class_name in dir(module): + class_ = getattr(module, class_name) + + if isclass(class_) and issubclass(class_, _Attribute) and class_ != _Attribute: + ATTRIBUTE_CLASSES.append(class_) + + +ATTRIBUTE_CLASSES = [] +_collect_attribute_classes() diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py new file mode 100644 index 0000000..0a9ead5 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _BooleanAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "boolean" + + @classmethod + def fromdict(cls, data_dict: dict): + return _BooleanAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != bool: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'bool')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py new file mode 100644 index 0000000..19931f0 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _IntegerAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "integer" + + @classmethod + def fromdict(cls, data_dict: dict): + return _IntegerAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != int: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'int')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py new file mode 100644 index 0000000..68faa59 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py @@ -0,0 +1,51 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _MultiSelectAttribute(_Attribute): + + options: set[str] + + @classmethod + def supports(cls, data_dict: dict): + return ( + type(data_dict) == dict and "type" in data_dict and data_dict["type"] == "multi-select" + ) + + @classmethod + def fromdict(cls, data_dict: dict): + return _MultiSelectAttribute(options=set(data_dict["options"])) + + def check(self, attribute_name: str, attribute_values, annotation_id: str) -> t.List[str]: + + if type(attribute_values) != list: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_values.__class__.__name__}' (should be 'list')." + ] + + for attribute_value in attribute_values: + if attribute_value not in self.options: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} has an undefined " + + f"value '{attribute_value}' (defined options: {self._stringify_options()})." + ] + + return [] + + def _stringify_options(self) -> str: + options_str = "" + + for option in sorted(list(self.options)): + options_str += f"'{option}', " + + if options_str != "": + options_str = options_str[:-2] + + return options_str diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py new file mode 100644 index 0000000..ecf5c2a --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py @@ -0,0 +1,50 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _SingleSelectAttribute(_Attribute): + + options: set[str] + + @classmethod + def supports(cls, data_dict: dict): + return ( + type(data_dict) == dict and "type" in data_dict and data_dict["type"] == "single-select" + ) + + @classmethod + def fromdict(cls, data_dict: dict): + return _SingleSelectAttribute(options=set(data_dict["options"])) + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + + if type(attribute_value) != str: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'str')." + ] + + if attribute_value not in self.options: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} has an undefined " + + f"value '{attribute_value}' (defined options: {self._stringify_options()})." + ] + + return [] + + def _stringify_options(self) -> str: + options_str = "" + + for option in sorted(list(self.options)): + options_str += f"'{option}', " + + if options_str != "": + options_str = options_str[:-2] + + return options_str diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py new file mode 100644 index 0000000..1aef07f --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _StringAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "string" + + @classmethod + def fromdict(cls, data_dict: dict): + return _StringAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != str: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'str')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py new file mode 100644 index 0000000..3d9deaa --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _VectorAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "vector" + + @classmethod + def fromdict(cls, data_dict: dict): + return _VectorAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != list: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'list')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py new file mode 100644 index 0000000..c335237 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py @@ -0,0 +1,113 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +import raillabel + +from ._attributes._attribute_abc import _Attribute, attribute_classes +from ._sensor_type import _SensorType + + +@dataclass +class _ObjectClass: + attributes: dict[str, t.Type[_Attribute]] + sensor_types: dict[raillabel.format.SensorType, _SensorType] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_ObjectClass": + + if "attributes" not in data_dict: + data_dict["attributes"] = {} + + if "sensor_types" not in data_dict: + data_dict["sensor_types"] = {} + + return _ObjectClass( + attributes={ + attr_name: cls._attribute_fromdict(attr) + for attr_name, attr in data_dict["attributes"].items() + }, + sensor_types=cls._sensor_types_fromdict(data_dict["sensor_types"]), + ) + + def check(self, annotation: t.Type[raillabel.format._ObjectAnnotation]) -> t.List[str]: + errors = [] + + errors.extend(self._check_undefined_attributes(annotation)) + errors.extend(self._check_missing_attributes(annotation)) + errors.extend(self._check_false_attribute_type(annotation)) + + return errors + + @classmethod + def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: + + for attribute_class in attribute_classes(): + if attribute_class.supports(attribute): + return attribute_class.fromdict(attribute) + + raise ValueError() + + @classmethod + def _sensor_types_fromdict(cls, sensor_types_dict: dict) -> dict[str, _SensorType]: + sensor_types = {} + + for type_id, sensor_type_dict in sensor_types_dict.items(): + sensor_types[raillabel.format.SensorType(type_id)] = _SensorType.fromdict( + sensor_type_dict + ) + + return sensor_types + + def _check_undefined_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + applicable_attributes = self._compile_applicable_attributes(annotation) + + for attr_name in annotation.attributes.keys(): + if attr_name not in applicable_attributes: + errors.append(f"Undefined attribute '{attr_name}' in annotation {annotation.uid}.") + + return errors + + def _check_missing_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + for attr_name in self._compile_applicable_attributes(annotation): + if attr_name not in annotation.attributes: + errors.append(f"Missing attribute '{attr_name}' in annotation {annotation.uid}.") + + return errors + + def _check_false_attribute_type( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + applicable_attributes = self._compile_applicable_attributes(annotation) + for attr_name, attr_value in annotation.attributes.items(): + if attr_name not in applicable_attributes: + continue + + errors.extend( + applicable_attributes[attr_name].check(attr_name, attr_value, annotation.uid) + ) + + return errors + + def _compile_applicable_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> dict[str, t.Type[_Attribute]]: + + applicable_attributes = self.attributes + + if annotation.sensor.type in self.sensor_types: + applicable_attributes.update(self.sensor_types[annotation.sensor.type].attributes) + + return applicable_attributes diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py new file mode 100644 index 0000000..9d74fba --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py @@ -0,0 +1,46 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +import raillabel + +from ._object_classes import _ObjectClass + + +@dataclass +class _Onthology: + classes: dict[str, _ObjectClass] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_Onthology": + return _Onthology( + {class_id: _ObjectClass.fromdict(class_) for class_id, class_ in data_dict.items()} + ) + + def check(self, scene: raillabel.Scene) -> t.List[str]: + self.errors = [] + + self._check_class_validity(scene) + annotations = self._compile_annotations(scene) + for annotation in annotations: + self.errors.extend(self.classes[annotation.object.type].check(annotation)) + + return self.errors + + def _check_class_validity(self, scene: raillabel.Scene) -> t.List[str]: + object_classes_in_scene = [obj.type for obj in scene.objects.values()] + + for object_class in object_classes_in_scene: + if object_class not in self.classes: + self.errors.append(f"Object type '{object_class}' is not defined.") + + def _compile_annotations( + self, scene: raillabel.Scene + ) -> list[t.Type[raillabel.format._ObjectAnnotation]]: + annotations = [] + for frame in scene.frames.values(): + annotations.extend(list(frame.annotations.values())) + + return annotations diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py new file mode 100644 index 0000000..7bf7258 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py @@ -0,0 +1,34 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attributes._attribute_abc import _Attribute, attribute_classes + + +@dataclass +class _SensorType: + attributes: dict[str, t.Type[_Attribute]] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_SensorType": + + if "attributes" not in data_dict: + data_dict["attributes"] = {} + + return _SensorType( + attributes={ + attr_name: cls._attribute_fromdict(attr) + for attr_name, attr in data_dict["attributes"].items() + } + ) + + @classmethod + def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: + + for attribute_class in attribute_classes(): + if attribute_class.supports(attribute): + return attribute_class.fromdict(attribute) + + raise ValueError() diff --git a/raillabel_providerkit/validation/validate_onthology/validate_onthology.py b/raillabel_providerkit/validation/validate_onthology/validate_onthology.py new file mode 100644 index 0000000..95bb40a --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/validate_onthology.py @@ -0,0 +1,66 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from pathlib import Path + +import jsonschema +import raillabel +import yaml + +from ...exceptions import OnthologySchemaError +from ._onthology_classes._onthology import _Onthology + + +def validate_onthology(scene: raillabel.Scene, onthology: t.Union[dict, Path]) -> t.List[str]: + """Validate a scene based on the classes and attributes. + + Parameters + ---------- + scene : raillabel.Scene + The scene containing the annotations. + onthology : dict or Path + Onthology YAML-data or file containing a information about all classes and their + attributes. The onthology must adhere to the onthology_schema. If a path is provided, the + file is loaded as a YAML. + + Returns + ------- + list[str] + list of all onthology errors in the scene. If an empty list is returned, then there are no + errors present. + """ + + if isinstance(onthology, Path): + onthology = _load_onthology(Path(onthology)) + + _validate_onthology_schema(onthology) + + onthology = _Onthology.fromdict(onthology) + + return onthology.check(scene) + + +def _load_onthology(path: Path) -> dict: + with path.open() as f: + onthology = yaml.safe_load(f) + return onthology + + +def _validate_onthology_schema(onthology: dict): + SCHEMA_PATH = Path(__file__).parent / "onthology_schema_v1.yaml" + + with SCHEMA_PATH.open() as f: + onthology_schema = yaml.safe_load(f) + + validator = jsonschema.Draft7Validator(schema=onthology_schema) + + schema_errors = "" + for error in validator.iter_errors(onthology): + schema_errors += f"${error.json_path[1:]}: {error.message}\n" + + if schema_errors != "": + raise OnthologySchemaError( + "The provided onthology is not valid. The following errors have been found:\n" + + schema_errors + ) diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py index 85685b4..8d9ad12 100644 --- a/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import typing as t from pathlib import Path import jsonschema @@ -30,7 +31,7 @@ def schema(schema_path) -> dict: def validator(schema) -> jsonschema.Draft7Validator: return jsonschema.Draft7Validator(schema) -def schema_errors(data: dict, validator: jsonschema.Draft7Validator) -> list[str]: +def schema_errors(data: dict, validator: jsonschema.Draft7Validator) -> t.List[str]: errors = [] for error in validator.iter_errors(data): diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py new file mode 100644 index 0000000..d207464 --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py @@ -0,0 +1,705 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import typing as t +from pathlib import Path +from uuid import uuid4 + +import pytest +import raillabel + +sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent)) +from raillabel_providerkit import exceptions +from raillabel_providerkit.validation import validate_onthology + +# == Helpers ========================== + +def make_dict_with_uids(objects: list) -> dict: + return {obj.uid: obj for obj in objects} + +def build_scene( + sensors: list[raillabel.format.Sensor], + objects: list[raillabel.format.Object], + annotations: list[t.Type[raillabel.format._ObjectAnnotation]] +) -> raillabel.Scene: + if type(sensors) == list: + sensors = make_dict_with_uids(sensors) + + return raillabel.Scene( + metadata=raillabel.format.Metadata(schema_version="1.0.0"), + sensors=sensors, + objects=make_dict_with_uids(objects), + frames={ + 0: raillabel.format.Frame( + uid=0, + annotations=make_dict_with_uids(annotations) + ) + } + ) + +@pytest.fixture +def metadata(): + return raillabel.format.Metadata(schema_version="1.0.0") + +@pytest.fixture +def sensors() -> list[raillabel.format.Sensor]: + return { + "rgb_middle": raillabel.format.Sensor( + uid="rgb_middle", + type=raillabel.format.SensorType.CAMERA, + ), + "lidar": raillabel.format.Sensor( + uid="lidar", + type=raillabel.format.SensorType.LIDAR, + ), + "radar": raillabel.format.Sensor( + uid="radar", + type=raillabel.format.SensorType.RADAR, + ), + } + +@pytest.fixture +def object_person() -> raillabel.format.Object: + return raillabel.format.Object( + uid="973ecc31-36f3-4b41-a1d8-9b584f265822", + name="person_0000", + type="person", + ) + +def build_object(type: str) -> raillabel.format.Object: + return raillabel.format.Object( + uid=uuid4, + name=type, + type=type, + ) + +def build_annotation( + object: raillabel.format.Object, + uid: str="a3f3abe5-082d-42ce-966c-bae9c6dae9d9", + sensor: raillabel.format.Sensor=raillabel.format.Sensor( + uid="rgb_middle", + type=raillabel.format.SensorType.CAMERA, + ), + attributes: dict={} +) -> raillabel.format.Bbox: + return raillabel.format.Bbox( + uid=uid, + object=object, + sensor=sensor, + attributes=attributes, + pos=[], + size=[], + ) + +# == Tests ============================ + +def test_onthology_schema_invalid(): + onthology = { + "person": { + "INVALID_FIELD": {} + } + } + + with pytest.raises(exceptions.OnthologySchemaError): + validate_onthology(None, onthology) + + +def test_valid_classes(metadata): + onthology = { + "person": {}, + "train": {}, + } + + scene = raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("person"), + build_object("train"), + ]) + ) + + assert validate_onthology(scene, onthology) == [] + +def test_invalid_class(metadata): + onthology = { + "person": {}, + "train": {}, + } + + scene = raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("UNDEFINED_CLASS"), + ]) + ) + + assert validate_onthology(scene, onthology) == [ + "Object type 'UNDEFINED_CLASS' is not defined." + ] + + +def test_undefined_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": {} + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "UNKNOWN_ATTRIBUTE": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Undefined attribute 'UNKNOWN_ATTRIBUTE' in annotation {annotation.uid}." + ] + +def test_missing_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={} + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Missing attribute 'number_of_fingers' in annotation {annotation.uid}." + ] + + +def test_valid_integer_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_integer_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": "THIS SHOULD BE AN INTEGER" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'number_of_fingers' of annotation {annotation.uid} is of type 'str' (should be 'int')." + ] + +def test_valid_string_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": "Gudrun" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_string_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": 42 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'first_name' of annotation {annotation.uid} is of type 'int' (should be 'str')." + ] + +def test_valid_boolean_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "has_cool_blue_shirt": "boolean" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "has_cool_blue_shirt": False + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_boolean_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "has_cool_blue_shirt": "boolean" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "has_cool_blue_shirt": "NO THE SHIRT IS ORANGE ... AND THIS SHOULD BE A BOOL" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'has_cool_blue_shirt' of annotation {annotation.uid} is of type 'str' (should be 'bool')." + ] + +def test_valid_vector_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "favorite_pizzas": "vector" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "favorite_pizzas": ["Diavolo", "Neapolitan", "Quattro Formaggi"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_vector_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "favorite_pizzas": "vector" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "favorite_pizzas": "does not like pizza (ikr)... THIS SHOULD BE A VECTOR AS WELL" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'favorite_pizzas' of annotation {annotation.uid} is of type 'str' (should be 'list')." + ] + +def test_valid_single_select_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "groceries" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_single_select_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": False + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} is of type 'bool' (should be 'str')." + ] + +def test_single_select_attribute_undefined_option(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "something very unexpected" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)')." + ] + +def test_valid_multi_select_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": ["groceries", "a baby"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_multi_select_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "a baby" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} is of type 'str' (should be 'list')." + ] + +def test_multi_select_attribute_undefined_option(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": ["a baby", "something very unexpected"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)')." + ] + +def test_multiple_attributes_valid(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer", + "first_name": "string", + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + } + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "groceries", + "number_of_fingers": 9, + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_multiple_attributes_invalid(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer", + "first_name": "string", + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + } + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "something very unexpected", + "number_of_fingers": 9, + "first_name": True, + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + c = validate_onthology(scene, onthology) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)').", + f"Attribute 'first_name' of annotation {annotation.uid} is of type 'bool' (should be 'str').", + ] + + +def test_valid_sensor_type_attribute(sensors, object_person): + onthology = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_invalid_sensor_type_attribute(sensors, object_person): + onthology = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": "None" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'number_of_fingers' of annotation {annotation.uid} is of type 'str' (should be 'int')." + ] + + +def test_valid_sensor_type_attributes_and_attributes(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + }, + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10, + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_invalid_sensor_type_attributes_and_attributes(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + }, + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Missing attribute 'number_of_fingers' in annotation {annotation.uid}." + ] + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"])