diff --git a/pyproject.toml b/pyproject.toml index 475c838..7ca671f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ classifiers = [ dependencies = [ "jsonschema>=4.4.0", "fastjsonschema>=2.16.2", - "raillabel>=3.1.0" + "raillabel>=3.1.0", + "pyyaml>=6.0.0" ] [project.urls] diff --git a/raillabel_providerkit/__init__.py b/raillabel_providerkit/__init__.py index a94acf2..733f88e 100644 --- a/raillabel_providerkit/__init__.py +++ b/raillabel_providerkit/__init__.py @@ -6,6 +6,7 @@ from . import format from .convert import loader_classes from .convert.convert import convert +from .validation.validate import validate try: __version__ = metadata.version("raillabel-providerkit") diff --git a/raillabel_providerkit/exceptions.py b/raillabel_providerkit/exceptions.py index 37319d8..47ff8f7 100644 --- a/raillabel_providerkit/exceptions.py +++ b/raillabel_providerkit/exceptions.py @@ -12,3 +12,9 @@ class SchemaError(Exception): """Raised when the data does not validate against a given schema.""" __module__ = "raillabel_providerkit" + + +class OnthologySchemaError(Exception): + """Raised when the .yaml-file provided is not valid against the schema.""" + + __module__ = "raillabel_providerkit" diff --git a/raillabel_providerkit/validate/raillabel_schema.json b/raillabel_providerkit/format/raillabel_schema.json similarity index 100% rename from raillabel_providerkit/validate/raillabel_schema.json rename to raillabel_providerkit/format/raillabel_schema.json diff --git a/raillabel_providerkit/validate/raillabel_schema.json.license b/raillabel_providerkit/format/raillabel_schema.json.license similarity index 100% rename from raillabel_providerkit/validate/raillabel_schema.json.license rename to raillabel_providerkit/format/raillabel_schema.json.license diff --git a/raillabel_providerkit/format/understand_ai/metadata.py b/raillabel_providerkit/format/understand_ai/metadata.py index 7d5fbf0..59b954e 100644 --- a/raillabel_providerkit/format/understand_ai/metadata.py +++ b/raillabel_providerkit/format/understand_ai/metadata.py @@ -82,7 +82,7 @@ def to_raillabel(self) -> dict: def _get_subschema_version(self) -> str: RAILLABEL_SCHEMA_PATH = ( - Path(__file__).parent.parent.parent / "validate" / "raillabel_schema.json" + Path(__file__).parent.parent.parent / "format" / "raillabel_schema.json" ) with RAILLABEL_SCHEMA_PATH.open() as schema_file: diff --git a/raillabel_providerkit/validate/__init__.py b/raillabel_providerkit/validation/__init__.py similarity index 68% rename from raillabel_providerkit/validate/__init__.py rename to raillabel_providerkit/validation/__init__.py index 4c38e62..098b319 100644 --- a/raillabel_providerkit/validate/__init__.py +++ b/raillabel_providerkit/validation/__init__.py @@ -1,3 +1,5 @@ # Copyright DB Netz AG and contributors # SPDX-License-Identifier: Apache-2.0 """Package for validating raillabel data regarding the format requirements.""" + +from .validate_onthology.validate_onthology import validate_onthology diff --git a/raillabel_providerkit/validation/validate.py b/raillabel_providerkit/validation/validate.py new file mode 100644 index 0000000..a3dddde --- /dev/null +++ b/raillabel_providerkit/validation/validate.py @@ -0,0 +1,35 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from pathlib import Path + +import raillabel + +from . import validate_onthology + + +def validate(scene: raillabel.Scene, onthology: t.Union[dict, Path]) -> t.List[str]: + """Validate a scene based on the Deutsche Bahn Requirements. + + Parameters + ---------- + scene : raillabel.Scene + The scene containing the annotations. + onthology : dict or Path + Onthology YAML-data or file containing a information about all classes and their + attributes. The onthology must adhere to the onthology_schema. If a path is provided, the + file is loaded as a YAML. + + Returns + ------- + list[str] + list of all requirement errors in the scene. If an empty list is returned, then there are + no errors present and the scene is valid. + """ + + errors = [] + + errors += validate_onthology(scene, onthology) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/__init__.py b/raillabel_providerkit/validation/validate_onthology/__init__.py new file mode 100644 index 0000000..56bbc60 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/__init__.py @@ -0,0 +1,3 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 +"""Package for validating a scene via an onthology.""" diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py new file mode 100644 index 0000000..aabc64e --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py @@ -0,0 +1,2 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py new file mode 100644 index 0000000..aabc64e --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py @@ -0,0 +1,2 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py new file mode 100644 index 0000000..6edebc5 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py @@ -0,0 +1,54 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import typing as t +from dataclasses import dataclass +from importlib import import_module +from inspect import isclass +from pathlib import Path +from pkgutil import iter_modules + + +@dataclass +class _Attribute(abc.ABC): + @classmethod + @abc.abstractmethod + def supports(cls, data_dict: dict) -> bool: + raise NotImplementedError + + @classmethod + @abc.abstractmethod + def fromdict(cls, data_dict: dict) -> t.Type["_Attribute"]: + raise NotImplementedError + + @abc.abstractmethod + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + raise NotImplementedError + + +def attribute_classes() -> t.List[t.Type[_Attribute]]: + """Return dictionary with Attribute child classes.""" + return ATTRIBUTE_CLASSES + + +def _collect_attribute_classes(): + """Collect attribute child classes and store them.""" + + global ATTRIBUTE_CLASSES + + package_dir = str(Path(__file__).resolve().parent) + for (_, module_name, _) in iter_modules([package_dir]): + + module = import_module( + f"raillabel_providerkit.validation.validate_onthology._onthology_classes._attributes.{module_name}" + ) + for class_name in dir(module): + class_ = getattr(module, class_name) + + if isclass(class_) and issubclass(class_, _Attribute) and class_ != _Attribute: + ATTRIBUTE_CLASSES.append(class_) + + +ATTRIBUTE_CLASSES = [] +_collect_attribute_classes() diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py new file mode 100644 index 0000000..0a9ead5 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _BooleanAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "boolean" + + @classmethod + def fromdict(cls, data_dict: dict): + return _BooleanAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != bool: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'bool')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py new file mode 100644 index 0000000..19931f0 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _IntegerAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "integer" + + @classmethod + def fromdict(cls, data_dict: dict): + return _IntegerAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != int: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'int')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py new file mode 100644 index 0000000..5805194 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py @@ -0,0 +1,51 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _MultiSelectAttribute(_Attribute): + + options: t.Set[str] + + @classmethod + def supports(cls, data_dict: dict): + return ( + type(data_dict) == dict and "type" in data_dict and data_dict["type"] == "multi-select" + ) + + @classmethod + def fromdict(cls, data_dict: dict): + return _MultiSelectAttribute(options=set(data_dict["options"])) + + def check(self, attribute_name: str, attribute_values, annotation_id: str) -> t.List[str]: + + if type(attribute_values) != list: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_values.__class__.__name__}' (should be 'list')." + ] + + for attribute_value in attribute_values: + if attribute_value not in self.options: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} has an undefined " + + f"value '{attribute_value}' (defined options: {self._stringify_options()})." + ] + + return [] + + def _stringify_options(self) -> str: + options_str = "" + + for option in sorted(list(self.options)): + options_str += f"'{option}', " + + if options_str != "": + options_str = options_str[:-2] + + return options_str diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py new file mode 100644 index 0000000..7ce4309 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py @@ -0,0 +1,50 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _SingleSelectAttribute(_Attribute): + + options: t.Set[str] + + @classmethod + def supports(cls, data_dict: dict): + return ( + type(data_dict) == dict and "type" in data_dict and data_dict["type"] == "single-select" + ) + + @classmethod + def fromdict(cls, data_dict: dict): + return _SingleSelectAttribute(options=set(data_dict["options"])) + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + + if type(attribute_value) != str: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'str')." + ] + + if attribute_value not in self.options: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} has an undefined " + + f"value '{attribute_value}' (defined options: {self._stringify_options()})." + ] + + return [] + + def _stringify_options(self) -> str: + options_str = "" + + for option in sorted(list(self.options)): + options_str += f"'{option}', " + + if options_str != "": + options_str = options_str[:-2] + + return options_str diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py new file mode 100644 index 0000000..1aef07f --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _StringAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "string" + + @classmethod + def fromdict(cls, data_dict: dict): + return _StringAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != str: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'str')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py new file mode 100644 index 0000000..3d9deaa --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _VectorAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "vector" + + @classmethod + def fromdict(cls, data_dict: dict): + return _VectorAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != list: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'list')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py new file mode 100644 index 0000000..b691f82 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py @@ -0,0 +1,113 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +import raillabel + +from ._attributes._attribute_abc import _Attribute, attribute_classes +from ._sensor_type import _SensorType + + +@dataclass +class _ObjectClass: + attributes: t.Dict[str, t.Type[_Attribute]] + sensor_types: t.Dict[raillabel.format.SensorType, _SensorType] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_ObjectClass": + + if "attributes" not in data_dict: + data_dict["attributes"] = {} + + if "sensor_types" not in data_dict: + data_dict["sensor_types"] = {} + + return _ObjectClass( + attributes={ + attr_name: cls._attribute_fromdict(attr) + for attr_name, attr in data_dict["attributes"].items() + }, + sensor_types=cls._sensor_types_fromdict(data_dict["sensor_types"]), + ) + + def check(self, annotation: t.Type[raillabel.format._ObjectAnnotation]) -> t.List[str]: + errors = [] + + errors.extend(self._check_undefined_attributes(annotation)) + errors.extend(self._check_missing_attributes(annotation)) + errors.extend(self._check_false_attribute_type(annotation)) + + return errors + + @classmethod + def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: + + for attribute_class in attribute_classes(): + if attribute_class.supports(attribute): + return attribute_class.fromdict(attribute) + + raise ValueError() + + @classmethod + def _sensor_types_fromdict(cls, sensor_types_dict: dict) -> t.Dict[str, _SensorType]: + sensor_types = {} + + for type_id, sensor_type_dict in sensor_types_dict.items(): + sensor_types[raillabel.format.SensorType(type_id)] = _SensorType.fromdict( + sensor_type_dict + ) + + return sensor_types + + def _check_undefined_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + applicable_attributes = self._compile_applicable_attributes(annotation) + + for attr_name in annotation.attributes.keys(): + if attr_name not in applicable_attributes: + errors.append(f"Undefined attribute '{attr_name}' in annotation {annotation.uid}.") + + return errors + + def _check_missing_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + for attr_name in self._compile_applicable_attributes(annotation): + if attr_name not in annotation.attributes: + errors.append(f"Missing attribute '{attr_name}' in annotation {annotation.uid}.") + + return errors + + def _check_false_attribute_type( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + applicable_attributes = self._compile_applicable_attributes(annotation) + for attr_name, attr_value in annotation.attributes.items(): + if attr_name not in applicable_attributes: + continue + + errors.extend( + applicable_attributes[attr_name].check(attr_name, attr_value, annotation.uid) + ) + + return errors + + def _compile_applicable_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.Dict[str, t.Type[_Attribute]]: + + applicable_attributes = self.attributes + + if annotation.sensor.type in self.sensor_types: + applicable_attributes.update(self.sensor_types[annotation.sensor.type].attributes) + + return applicable_attributes diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py new file mode 100644 index 0000000..456f0b2 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py @@ -0,0 +1,47 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +import raillabel + +from ._object_classes import _ObjectClass + + +@dataclass +class _Onthology: + classes: t.Dict[str, _ObjectClass] + errors = [] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_Onthology": + return _Onthology( + {class_id: _ObjectClass.fromdict(class_) for class_id, class_ in data_dict.items()} + ) + + def check(self, scene: raillabel.Scene) -> t.List[str]: + self.errors = [] + + self._check_class_validity(scene) + annotations = self._compile_annotations(scene) + for annotation in annotations: + self.errors.extend(self.classes[annotation.object.type].check(annotation)) + + return self.errors + + def _check_class_validity(self, scene: raillabel.Scene) -> t.List[str]: + object_classes_in_scene = [obj.type for obj in scene.objects.values()] + + for object_class in object_classes_in_scene: + if object_class not in self.classes: + self.errors.append(f"Object type '{object_class}' is not defined.") + + def _compile_annotations( + self, scene: raillabel.Scene + ) -> t.List[t.Type[raillabel.format._ObjectAnnotation]]: + annotations = [] + for frame in scene.frames.values(): + annotations.extend(list(frame.annotations.values())) + + return annotations diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py new file mode 100644 index 0000000..a74960b --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py @@ -0,0 +1,34 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attributes._attribute_abc import _Attribute, attribute_classes + + +@dataclass +class _SensorType: + attributes: t.Dict[str, t.Type[_Attribute]] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_SensorType": + + if "attributes" not in data_dict: + data_dict["attributes"] = {} + + return _SensorType( + attributes={ + attr_name: cls._attribute_fromdict(attr) + for attr_name, attr in data_dict["attributes"].items() + } + ) + + @classmethod + def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: + + for attribute_class in attribute_classes(): + if attribute_class.supports(attribute): + return attribute_class.fromdict(attribute) + + raise ValueError() diff --git a/raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml b/raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml new file mode 100644 index 0000000..8b3db11 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml @@ -0,0 +1,87 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +"$schema": http://json-schema.org/draft-07/schema# +version: 1.0.0 + +definitions: + + attribute: + oneOf: + [ + "$ref": "#/definitions/boolean_attribute", + "$ref": "#/definitions/integer_attribute", + "$ref": "#/definitions/multi_select_attribute", + "$ref": "#/definitions/single_select_attribute", + "$ref": "#/definitions/string_attribute", + "$ref": "#/definitions/vector_attribute", + ] + + boolean_attribute: + const: boolean + + class: + additionalProperties: false + properties: + attributes: + additionalProperties: false + patternProperties: + "^": + "$ref": "#/definitions/attribute" + type: object + sensor_types: + additionalProperties: false + patternProperties: + "^(camera|lidar|radar)$": + "$ref": "#/definitions/sensor_type" + type: object + type: object + + integer_attribute: + const: integer + + single_select_attribute: + additionalProperties: false + properties: + type: + const: single-select + options: + type: array + items: + type: string + type: object + + multi_select_attribute: + additionalProperties: false + properties: + type: + const: multi-select + options: + type: array + items: + type: string + type: object + + sensor_type: + additionalProperties: false + properties: + attributes: + additionalProperties: false + patternProperties: + "^": + "$ref": "#/definitions/attribute" + type: object + type: object + + string_attribute: + const: string + + vector_attribute: + const: vector + +additionalProperties: false +patternProperties: + "^": + "$ref": "#/definitions/class" + +type: object diff --git a/raillabel_providerkit/validation/validate_onthology/validate_onthology.py b/raillabel_providerkit/validation/validate_onthology/validate_onthology.py new file mode 100644 index 0000000..95bb40a --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/validate_onthology.py @@ -0,0 +1,66 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from pathlib import Path + +import jsonschema +import raillabel +import yaml + +from ...exceptions import OnthologySchemaError +from ._onthology_classes._onthology import _Onthology + + +def validate_onthology(scene: raillabel.Scene, onthology: t.Union[dict, Path]) -> t.List[str]: + """Validate a scene based on the classes and attributes. + + Parameters + ---------- + scene : raillabel.Scene + The scene containing the annotations. + onthology : dict or Path + Onthology YAML-data or file containing a information about all classes and their + attributes. The onthology must adhere to the onthology_schema. If a path is provided, the + file is loaded as a YAML. + + Returns + ------- + list[str] + list of all onthology errors in the scene. If an empty list is returned, then there are no + errors present. + """ + + if isinstance(onthology, Path): + onthology = _load_onthology(Path(onthology)) + + _validate_onthology_schema(onthology) + + onthology = _Onthology.fromdict(onthology) + + return onthology.check(scene) + + +def _load_onthology(path: Path) -> dict: + with path.open() as f: + onthology = yaml.safe_load(f) + return onthology + + +def _validate_onthology_schema(onthology: dict): + SCHEMA_PATH = Path(__file__).parent / "onthology_schema_v1.yaml" + + with SCHEMA_PATH.open() as f: + onthology_schema = yaml.safe_load(f) + + validator = jsonschema.Draft7Validator(schema=onthology_schema) + + schema_errors = "" + for error in validator.iter_errors(onthology): + schema_errors += f"${error.json_path[1:]}: {error.message}\n" + + if schema_errors != "": + raise OnthologySchemaError( + "The provided onthology is not valid. The following errors have been found:\n" + + schema_errors + ) diff --git a/tests/conftest.py b/tests/conftest.py index cf0802b..f2f58e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ json_data_directories = [ Path(__file__).parent / "__test_assets__", - Path(__file__).parent.parent / "raillabel_providerkit" / "validate" + Path(__file__).parent.parent / "raillabel_providerkit" / "format" ] @pytest.fixture diff --git a/tests/test_raillabel_providerkit/validate/test_raillabel_v2_schema.py b/tests/test_raillabel_providerkit/format/test_raillabel_v2_schema.py similarity index 100% rename from tests/test_raillabel_providerkit/validate/test_raillabel_v2_schema.py rename to tests/test_raillabel_providerkit/format/test_raillabel_v2_schema.py diff --git a/tests/test_raillabel_providerkit/validation/conftest.py b/tests/test_raillabel_providerkit/validation/conftest.py new file mode 100644 index 0000000..cfdf202 --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/conftest.py @@ -0,0 +1,9 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +from validate_onthology.test_validate_onthology import ( + demo_onthology, + invalid_onthology_scene, + metadata, + valid_onthology_scene, +) diff --git a/tests/test_raillabel_providerkit/validation/test_validate.py b/tests/test_raillabel_providerkit/validation/test_validate.py new file mode 100644 index 0000000..e3ce73d --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/test_validate.py @@ -0,0 +1,27 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import typing as t +from pathlib import Path +from uuid import uuid4 + +import pytest +import raillabel + +sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent)) +from raillabel_providerkit import validate + +# == Tests ============================ + +def test_no_errors(demo_onthology, valid_onthology_scene): + assert validate(valid_onthology_scene, demo_onthology) == [] + +def test_onthology_errors(demo_onthology, invalid_onthology_scene): + assert len(validate(invalid_onthology_scene, demo_onthology)) == 1 + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"]) diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py new file mode 100644 index 0000000..8d9ad12 --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py @@ -0,0 +1,197 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import typing as t +from pathlib import Path + +import jsonschema +import pytest +import yaml + +# == Fixtures ========================= + +@pytest.fixture +def schema_path() -> Path: + return ( + Path(__file__).parent.parent.parent.parent.parent + / "raillabel_providerkit" + / "validation" + / "validate_onthology" + / "onthology_schema_v1.yaml" + ) + +@pytest.fixture +def schema(schema_path) -> dict: + with schema_path.open() as f: + schema_data = yaml.safe_load(f) + return schema_data + +@pytest.fixture +def validator(schema) -> jsonschema.Draft7Validator: + return jsonschema.Draft7Validator(schema) + +def schema_errors(data: dict, validator: jsonschema.Draft7Validator) -> t.List[str]: + errors = [] + + for error in validator.iter_errors(data): + errors.append("$" + error.json_path[1:] + ": " + str(error.message)) + + return errors + +# == Tests ========================= + +def test_classes(validator): + data = { + "person": {}, + "train": {}, + } + + assert schema_errors(data, validator) == [] + +def test_class_unsupported_field(validator): + data = { + "person": { + "UNSUPPORTED_FIELD": {} + } + } + + assert schema_errors(data, validator) == [ + "$.person: Additional properties are not allowed ('UNSUPPORTED_FIELD' was unexpected)", + ] + + +def test_attributes_field(validator): + data = { + "person": { + "attributes": {} + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_string(validator): + data = { + "person": { + "attributes": { + "name": "string" + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_integer(validator): + data = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_boolean(validator): + data = { + "person": { + "attributes": { + "number_of_fingers": "boolean" + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_single_select(validator): + data = { + "person": { + "attributes": { + "carrying": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the new Slicer-Dicer 3000 (WOW!)" + ] + } + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_multi_select(validator): + data = { + "person": { + "attributes": { + "carrying": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the new Slicer-Dicer 3000 (WOW!)" + ] + } + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_vector(validator): + data = { + "person": { + "attributes": { + "carrying": "vector" + } + } + } + + assert schema_errors(data, validator) == [] + + +def test_sensor_types(validator): + data = { + "person": { + "sensor_types": { + "camera": {}, + "lidar": {}, + "radar": {}, + } + } + } + + assert schema_errors(data, validator) == [] + +def test_sensor_types_unsupported_type(validator): + data = { + "person": { + "sensor_types": { + "UNSUPPORTED_SENSOR_TYPE": {}, + "lidar": {}, + } + } + } + + assert len(schema_errors(data, validator)) == 1 + +def test_sensor_type_attributes(validator): + data = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "name": "string" + } + }, + } + } + } + + assert schema_errors(data, validator) == [] + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"]) diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py new file mode 100644 index 0000000..53baa7f --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py @@ -0,0 +1,734 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import typing as t +from pathlib import Path +from uuid import uuid4 + +import pytest +import raillabel + +sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent)) +from raillabel_providerkit import exceptions +from raillabel_providerkit.validation import validate_onthology + +# == Helpers ========================== + +def make_dict_with_uids(objects: list) -> dict: + return {obj.uid: obj for obj in objects} + +def build_scene( + sensors: t.List[raillabel.format.Sensor], + objects: t.List[raillabel.format.Object], + annotations: t.List[t.Type[raillabel.format._ObjectAnnotation]] +) -> raillabel.Scene: + if type(sensors) == list: + sensors = make_dict_with_uids(sensors) + + return raillabel.Scene( + metadata=raillabel.format.Metadata(schema_version="1.0.0"), + sensors=sensors, + objects=make_dict_with_uids(objects), + frames={ + 0: raillabel.format.Frame( + uid=0, + annotations=make_dict_with_uids(annotations) + ) + } + ) + +@pytest.fixture +def sensors() -> t.List[raillabel.format.Sensor]: + return { + "rgb_middle": raillabel.format.Sensor( + uid="rgb_middle", + type=raillabel.format.SensorType.CAMERA, + ), + "lidar": raillabel.format.Sensor( + uid="lidar", + type=raillabel.format.SensorType.LIDAR, + ), + "radar": raillabel.format.Sensor( + uid="radar", + type=raillabel.format.SensorType.RADAR, + ), + } + +@pytest.fixture +def object_person() -> raillabel.format.Object: + return raillabel.format.Object( + uid="973ecc31-36f3-4b41-a1d8-9b584f265822", + name="person_0000", + type="person", + ) + +def build_object(type: str) -> raillabel.format.Object: + return raillabel.format.Object( + uid=uuid4, + name=type, + type=type, + ) + +def build_annotation( + object: raillabel.format.Object, + uid: str="a3f3abe5-082d-42ce-966c-bae9c6dae9d9", + sensor: raillabel.format.Sensor=raillabel.format.Sensor( + uid="rgb_middle", + type=raillabel.format.SensorType.CAMERA, + ), + attributes: dict={} +) -> raillabel.format.Bbox: + return raillabel.format.Bbox( + uid=uid, + object=object, + sensor=sensor, + attributes=attributes, + pos=[], + size=[], + ) + +# == Fixtures ========================= + +@pytest.fixture +def metadata(): + return raillabel.format.Metadata(schema_version="1.0.0") + +@pytest.fixture +def demo_onthology() -> dict: + return { + "person": {}, + "train": {}, + } + +@pytest.fixture +def valid_onthology_scene(metadata) -> raillabel.Scene: + return raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("person"), + build_object("train"), + ]) + ) + +@pytest.fixture +def invalid_onthology_scene(metadata) -> raillabel.Scene: + return raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("INVALID_CLASS"), + ]) + ) + +# == Tests ============================ + +def test_onthology_schema_invalid(): + onthology = { + "person": { + "INVALID_FIELD": {} + } + } + + with pytest.raises(exceptions.OnthologySchemaError): + validate_onthology(None, onthology) + + +def test_valid_classes(metadata): + onthology = { + "person": {}, + "train": {}, + } + + scene = raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("person"), + build_object("train"), + ]) + ) + + assert validate_onthology(scene, onthology) == [] + +def test_invalid_class(metadata): + onthology = { + "person": {}, + "train": {}, + } + + scene = raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("UNDEFINED_CLASS"), + ]) + ) + + assert validate_onthology(scene, onthology) == [ + "Object type 'UNDEFINED_CLASS' is not defined." + ] + + +def test_undefined_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": {} + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "UNKNOWN_ATTRIBUTE": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Undefined attribute 'UNKNOWN_ATTRIBUTE' in annotation {annotation.uid}." + ] + +def test_missing_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={} + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Missing attribute 'number_of_fingers' in annotation {annotation.uid}." + ] + + +def test_valid_integer_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_integer_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": "THIS SHOULD BE AN INTEGER" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'number_of_fingers' of annotation {annotation.uid} is of type 'str' (should be 'int')." + ] + +def test_valid_string_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": "Gudrun" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_string_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": 42 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'first_name' of annotation {annotation.uid} is of type 'int' (should be 'str')." + ] + +def test_valid_boolean_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "has_cool_blue_shirt": "boolean" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "has_cool_blue_shirt": False + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_boolean_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "has_cool_blue_shirt": "boolean" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "has_cool_blue_shirt": "NO THE SHIRT IS ORANGE ... AND THIS SHOULD BE A BOOL" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'has_cool_blue_shirt' of annotation {annotation.uid} is of type 'str' (should be 'bool')." + ] + +def test_valid_vector_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "favorite_pizzas": "vector" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "favorite_pizzas": ["Diavolo", "Neapolitan", "Quattro Formaggi"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_vector_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "favorite_pizzas": "vector" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "favorite_pizzas": "does not like pizza (ikr)... THIS SHOULD BE A VECTOR AS WELL" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'favorite_pizzas' of annotation {annotation.uid} is of type 'str' (should be 'list')." + ] + +def test_valid_single_select_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "groceries" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_single_select_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": False + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} is of type 'bool' (should be 'str')." + ] + +def test_single_select_attribute_undefined_option(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "something very unexpected" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)')." + ] + +def test_valid_multi_select_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": ["groceries", "a baby"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_multi_select_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "a baby" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} is of type 'str' (should be 'list')." + ] + +def test_multi_select_attribute_undefined_option(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": ["a baby", "something very unexpected"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)')." + ] + +def test_multiple_attributes_valid(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer", + "first_name": "string", + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + } + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "groceries", + "number_of_fingers": 9, + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_multiple_attributes_invalid(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer", + "first_name": "string", + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + } + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "something very unexpected", + "number_of_fingers": 9, + "first_name": True, + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + c = validate_onthology(scene, onthology) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)').", + f"Attribute 'first_name' of annotation {annotation.uid} is of type 'bool' (should be 'str').", + ] + + +def test_valid_sensor_type_attribute(sensors, object_person): + onthology = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_invalid_sensor_type_attribute(sensors, object_person): + onthology = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": "None" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'number_of_fingers' of annotation {annotation.uid} is of type 'str' (should be 'int')." + ] + + +def test_valid_sensor_type_attributes_and_attributes(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + }, + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10, + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_invalid_sensor_type_attributes_and_attributes(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + }, + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Missing attribute 'number_of_fingers' in annotation {annotation.uid}." + ] + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"])