From f4029b86bca11763b067a6007e85429e372aad17 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Fri, 28 Jul 2023 17:29:49 +0200 Subject: [PATCH 01/34] base pydantic code generator --- .gitignore | 1 + pydantic/Base.py | 31 +++ pydantic/PositionPoint.py | 74 +++++++ pydantic/__init__.py | 1 + pydantic/langPack.py | 181 ++++++++++++++++++ .../pydantic_class_template.mustache | 29 +++ .../templates/pydantic_enum_template.mustache | 12 ++ pydantic/util.py | 18 ++ 8 files changed, 347 insertions(+) create mode 100644 pydantic/Base.py create mode 100644 pydantic/PositionPoint.py create mode 100644 pydantic/__init__.py create mode 100644 pydantic/langPack.py create mode 100644 pydantic/templates/pydantic_class_template.mustache create mode 100644 pydantic/templates/pydantic_enum_template.mustache create mode 100644 pydantic/util.py diff --git a/.gitignore b/.gitignore index 418f90a7..fe88a1ca 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ cimpy/* *.py[cod] __pycache__/ .vscode/launch.json +CGMES_2.4.15_27JAN2020_pydantic/ diff --git a/pydantic/Base.py b/pydantic/Base.py new file mode 100644 index 00000000..96385df6 --- /dev/null +++ b/pydantic/Base.py @@ -0,0 +1,31 @@ +from pydantic import BaseModel +from enum import IntEnum + + +class CgmesProfileEnum(IntEnum): + EQ = 0 + SSH = 1 + TP = 2 + SV = 3 + DY = 4 + GL = 5 + DL = 5 + TP_DB = 7 + ED_BD = 8 + +class Base(BaseModel): + """ + Base Class for CIM + """ + + class Config: + @staticmethod + def schema_extra(schema: dict, _): + props = {} + for k, v in schema.get('properties', {}).items(): + if not v.get("hidden", False): + props[k] = v + schema["properties"] = props + + def printxml(self, dict={}): + return dict diff --git a/pydantic/PositionPoint.py b/pydantic/PositionPoint.py new file mode 100644 index 00000000..e3fc6893 --- /dev/null +++ b/pydantic/PositionPoint.py @@ -0,0 +1,74 @@ +from __future__ import annotations +from .Base import Base, CgmesProfileEnum + +from pydantic import ( + ConfigDict, + computed_field, + field_validator, + Field +) + +from geoalchemy2.shape import to_shape +from geoalchemy2.elements import WKBElement +from typing import Optional, List + +from shapely.geometry import Point +from geoalchemy2.shape import to_shape +from .Location import Location + +class PositionPoint(Base): + ''' + Set of spatial coordinates that determine a point, defined in the coordinate system specified in 'Location.CoordinateSystem'. Use a single position point instance to desribe a point-oriented location. Use a sequence of position points to describe a line-oriented object (physical location of non-point oriented objects like cables or lines), or area of an object (like a substation or a geographical zone - in this case, have first and last position point with the same values). + + :Location: Location described by this position point. + :sequenceNumber: Zero-relative sequence number of this point within a series of points. + :xPosition: X axis position. + :yPosition: Y axis position. + :zPosition: (if applicable) Z axis position. + ''' + + possibleProfileList: dict = Field(default={'class': [CgmesProfileEnum.GL, ], + 'Location': [CgmesProfileEnum.GL, ], + 'sequenceNumber': [CgmesProfileEnum.GL, ], + 'xPosition': [CgmesProfileEnum.GL, ], + 'yPosition': [CgmesProfileEnum.GL, ], + 'zPosition': [CgmesProfileEnum.GL, ], + }, hidden=True) + + Location: List[Location] + sequenceNumber: Optional[int] + point: Point # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + + model_config = ConfigDict(from_attributes=True) + + @computed_field + @property + def xPosition(self) -> str: + return str(self.point.x) + + @computed_field + @property + def yPosition(self) -> str: + return str(self.point.y) + + @computed_field + @property + def zPosition(self) -> str: + return str(self.point.z) + + # arbitrary_types_allowed is used because shapely data types are not based + # on Pydantic ones, so model mapping is not native. + model_config = ConfigDict(from_attributes=True, arbitrary_types_allowed=True) + + # Pydantic needs help to map GeoAlchemy classes to Shapely + @field_validator("point", mode="before") + def validate_point_format(cls, v): + if isinstance(v, Point): + return v + elif isinstance(v, WKBElement): + point = to_shape(v) + if point.geom_type != "Point": + raise ValueError("must be a Point") + return Point(point) + else: + raise ValueError("must be a Point or a WKBElement") diff --git a/pydantic/__init__.py b/pydantic/__init__.py new file mode 100644 index 00000000..1d69ab3e --- /dev/null +++ b/pydantic/__init__.py @@ -0,0 +1 @@ +import python.langPack diff --git a/pydantic/langPack.py b/pydantic/langPack.py new file mode 100644 index 00000000..792f98b3 --- /dev/null +++ b/pydantic/langPack.py @@ -0,0 +1,181 @@ +import os +import chevron +import logging +import glob +import shutil +import sys +logger = logging.getLogger(__name__) + +# This makes sure we have somewhere to write the classes, and +# creates a couple of files the python implementation needs. +# cgmes_profile_info details which uri belongs in each profile. +# We don't use that here because we aren't creating the header +# data for the separate profiles. +def setup(version_path, cgmes_profile_info): + if not os.path.exists(version_path): + os.makedirs(version_path) + _create_init(version_path) + _copy_files(version_path) + +def location(version): + return "pydantic." + version + ".Base"; + +base = { + "base_class": "Base", + "class_location": location +} + +template_files=[ { "filename": "pydantic_class_template.mustache", "ext": ".py" } ] +enum_template_files = [ { "filename": "pydantic_enum_template.mustache", "ext": ".py" } ] + +def get_class_location(class_name, class_map, version): + # Check if the current class has a parent class + if class_map[class_name].superClass(): + if class_map[class_name].superClass() in class_map: + return 'pydantic.' + version + "." + class_map[class_name].superClass() + elif class_map[class_name].superClass() == 'Base' or class_map[class_name].superClass() == None: + return location(version) + else: + return location(version) + +partials = {} + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_instances(text, render): + instance = eval(render(text)) + if "label" in instance: + value = instance['label'] + ' = "' + instance['label'] + '"' + if 'comment' in instance: + value += ' #' + instance['comment'] + return value + else: + return '' + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_imports(text, render): + rendered = render(text) + res = None + try: + res = eval(rendered) + finally: + result = '' + classes = set () + if res: + for val in res: + if "range" in val: + classes.add(val['range'].split('#')[1]) + for val in classes: + result += "from ."+val+" import "+val+"\n" + return result + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_default(text, render): + attribute = eval(render(text)) + + if "range" in attribute and "isFixed" in attribute: + return ' = ' + attribute['range'].split('#')[1] + '.' + attribute['isFixed'] + if "multiplicity" in attribute and attribute['multiplicity'] in ['M:0..n'] and "range" in attribute: + return ' = None' + if "range" not in attribute and "isFixed" in attribute: + return ' = "' + attribute['isFixed'] + '"' + return '' + +def _compute_data_type(attribute): + if 'label' in attribute and attribute['label'] == 'mRID': + return 'uuid.UUID' + + if "dataType" in attribute: + if attribute['dataType'].startswith('#'): + datatype = attribute['dataType'].split('#')[1] + if (datatype == 'Integer' or datatype == 'integer'): + return 'int' + if (datatype == 'Boolean'): + return 'bool' + if (datatype == 'String'): + return 'str' + if (datatype == 'DateTime'): + return 'str' + if (datatype == 'Date'): + return 'str' + if (datatype == 'String'): + return 'str' + else: + return "float" + if "range" in attribute: + return attribute['range'].split('#')[1] + +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_data_type(text, render): + attribute = eval(render(text)) + + datatype = _compute_data_type(attribute) + + if "multiplicity" in attribute: + multiplicity = attribute['multiplicity'] + if multiplicity in ['M:1..1', '']: + return datatype + if multiplicity in ['M:0..1']: + return 'Optional['+datatype+']' + elif multiplicity in ['M:0..n'] or 'M:0..' in multiplicity: + return 'Optional[List['+datatype+']]' + elif multiplicity in ['M:1', 'M:1..n'] or 'M:1..' in multiplicity: + return 'List['+datatype+']' + else: + return 'List['+datatype+']' + else: + return datatype + +def set_enum_classes(new_enum_classes): + return + +def set_float_classes(new_float_classes): + return + +def run_template(version_path, class_details): + if class_details['class_name'] in ['Float','Integer','String','Boolean','Date']: + templates = [] + elif class_details['has_instances'] == True: + templates = enum_template_files + else: + templates = template_files + + + for template_info in templates: + class_file = os.path.join(version_path, class_details['class_name'] + template_info["ext"]) + if not os.path.exists(class_file): + with open(class_file, 'w') as file: + template_path = os.path.join(os.getcwd(), 'pydantic/templates', template_info["filename"]) + class_details['setDefault'] = _set_default + class_details['setDataType'] = _set_data_type + class_details['setImports'] = _set_imports + class_details['setInstances'] = _set_instances + with open(template_path) as f: + args = { + 'data': class_details, + 'template': f, + 'partials_dict': partials + } + output = chevron.render(**args) + file.write(output) + + +def _create_init(path): + init_file = path + "/__init__.py" + with open(init_file, 'w'): + pass + +# creates the Base class file, all classes inherit from this class +def _copy_files(path): + shutil.copy("pydantic/Base.py", path + "/Base.py") + shutil.copy("pydantic/PositionPoint.py", path + "/PositionPoint.py") + shutil.copy("pydantic/util.py", path + "/util.py") + +def resolve_headers(path): + filenames = glob.glob(path + "/*.py") + include_names = [] + for filename in filenames: + include_names.append(os.path.splitext(os.path.basename(filename))[0]) + with open(path + "/__init__.py", "w") as header_file: + for include_name in include_names: + header_file.write("from " + "." + include_name + " import " + include_name + " as " + include_name + "\n") + header_file.close() diff --git a/pydantic/templates/pydantic_class_template.mustache b/pydantic/templates/pydantic_class_template.mustache new file mode 100644 index 00000000..917b7064 --- /dev/null +++ b/pydantic/templates/pydantic_class_template.mustache @@ -0,0 +1,29 @@ +from __future__ import annotations +from .{{sub_class_of}} import {{sub_class_of}} +from .Base import CgmesProfileEnum +import uuid +from pydantic import ( + ConfigDict, + Field +) +from typing import Optional, Iterator, List +{{#setImports}}{{attributes}}{{/setImports}} + +class {{class_name}}({{sub_class_of}}): + ''' + {{{class_comment}}} + + {{#attributes}} + :{{label}}: {{{comment}}} + {{/attributes}} + ''' + + possibleProfileList: dict = Field(default={'class': [{{#class_origin}}CgmesProfileEnum.{{origin}}, {{/class_origin}}], + {{#attributes}}'{{label}}': [{{#attr_origin}}CgmesProfileEnum.{{origin}}, {{/attr_origin}}], + {{/attributes}} }, hidden=True) + + {{#attributes}} + {{label}}: {{#setDataType}}{{.}}{{/setDataType}}{{#setDefault}}{{.}}{{/setDefault}} + {{/attributes}} + + model_config = ConfigDict(from_attributes=True) diff --git a/pydantic/templates/pydantic_enum_template.mustache b/pydantic/templates/pydantic_enum_template.mustache new file mode 100644 index 00000000..f48e9cca --- /dev/null +++ b/pydantic/templates/pydantic_enum_template.mustache @@ -0,0 +1,12 @@ +from .{{sub_class_of}} import {{sub_class_of}} +from enum import Enum + +class {{class_name}}(str,Enum): + + ''' + {{{class_comment}}} + ''' + + {{#instances}} + {{#setInstances}}{{.}}{{/setInstances}} + {{/instances}} diff --git a/pydantic/util.py b/pydantic/util.py new file mode 100644 index 00000000..e29622f8 --- /dev/null +++ b/pydantic/util.py @@ -0,0 +1,18 @@ +from contextlib import contextmanager +from pydantic import ( + ValidationError, +) +from typing import Iterator + +def is_recursion_validation_error(exc: ValidationError) -> bool: + errors = exc.errors() + return len(errors) == 1 and errors[0]["type"] == "recursion_loop" + + +@contextmanager +def suppress_recursion_validation_error() -> Iterator[None]: + try: + yield + except ValidationError as exc: + if not is_recursion_validation_error(exc): + raise exc From a89ad7a128e9cff3fc4a8f39e808b7d9a645c698 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Sat, 29 Jul 2023 09:44:55 +0200 Subject: [PATCH 02/34] fix __init__.py import --- pydantic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 1d69ab3e..9f57f01a 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -1 +1 @@ -import python.langPack +import pypdantic.langPack From d92eadce7d7ed9ca529244068a2e20b673d6ca1e Mon Sep 17 00:00:00 2001 From: Bot Date: Sat, 29 Jul 2023 06:32:12 +0000 Subject: [PATCH 03/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/Base.py | 5 +- pydantic/PositionPoint.py | 52 ++++++++----- pydantic/langPack.py | 158 +++++++++++++++++++++++--------------- pydantic/util.py | 1 + 4 files changed, 131 insertions(+), 85 deletions(-) diff --git a/pydantic/Base.py b/pydantic/Base.py index 96385df6..37c754a2 100644 --- a/pydantic/Base.py +++ b/pydantic/Base.py @@ -13,16 +13,17 @@ class CgmesProfileEnum(IntEnum): TP_DB = 7 ED_BD = 8 + class Base(BaseModel): """ Base Class for CIM """ - + class Config: @staticmethod def schema_extra(schema: dict, _): props = {} - for k, v in schema.get('properties', {}).items(): + for k, v in schema.get("properties", {}).items(): if not v.get("hidden", False): props[k] = v schema["properties"] = props diff --git a/pydantic/PositionPoint.py b/pydantic/PositionPoint.py index e3fc6893..c6c61e59 100644 --- a/pydantic/PositionPoint.py +++ b/pydantic/PositionPoint.py @@ -1,12 +1,7 @@ from __future__ import annotations from .Base import Base, CgmesProfileEnum -from pydantic import ( - ConfigDict, - computed_field, - field_validator, - Field -) +from pydantic import ConfigDict, computed_field, field_validator, Field from geoalchemy2.shape import to_shape from geoalchemy2.elements import WKBElement @@ -16,24 +11,41 @@ from geoalchemy2.shape import to_shape from .Location import Location + class PositionPoint(Base): - ''' + """ Set of spatial coordinates that determine a point, defined in the coordinate system specified in 'Location.CoordinateSystem'. Use a single position point instance to desribe a point-oriented location. Use a sequence of position points to describe a line-oriented object (physical location of non-point oriented objects like cables or lines), or area of an object (like a substation or a geographical zone - in this case, have first and last position point with the same values). - :Location: Location described by this position point. - :sequenceNumber: Zero-relative sequence number of this point within a series of points. - :xPosition: X axis position. - :yPosition: Y axis position. - :zPosition: (if applicable) Z axis position. - ''' + :Location: Location described by this position point. + :sequenceNumber: Zero-relative sequence number of this point within a series of points. + :xPosition: X axis position. + :yPosition: Y axis position. + :zPosition: (if applicable) Z axis position. + """ - possibleProfileList: dict = Field(default={'class': [CgmesProfileEnum.GL, ], - 'Location': [CgmesProfileEnum.GL, ], - 'sequenceNumber': [CgmesProfileEnum.GL, ], - 'xPosition': [CgmesProfileEnum.GL, ], - 'yPosition': [CgmesProfileEnum.GL, ], - 'zPosition': [CgmesProfileEnum.GL, ], - }, hidden=True) + possibleProfileList: dict = Field( + default={ + "class": [ + CgmesProfileEnum.GL, + ], + "Location": [ + CgmesProfileEnum.GL, + ], + "sequenceNumber": [ + CgmesProfileEnum.GL, + ], + "xPosition": [ + CgmesProfileEnum.GL, + ], + "yPosition": [ + CgmesProfileEnum.GL, + ], + "zPosition": [ + CgmesProfileEnum.GL, + ], + }, + hidden=True, + ) Location: List[Location] sequenceNumber: Optional[int] diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 792f98b3..e51f1a2d 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -4,8 +4,10 @@ import glob import shutil import sys + logger = logging.getLogger(__name__) + # This makes sure we have somewhere to write the classes, and # creates a couple of files the python implementation needs. # cgmes_profile_info details which uri belongs in each profile. @@ -17,39 +19,45 @@ def setup(version_path, cgmes_profile_info): _create_init(version_path) _copy_files(version_path) + def location(version): - return "pydantic." + version + ".Base"; + return "pydantic." + version + ".Base" + + +base = {"base_class": "Base", "class_location": location} -base = { - "base_class": "Base", - "class_location": location -} +template_files = [{"filename": "pydantic_class_template.mustache", "ext": ".py"}] +enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] -template_files=[ { "filename": "pydantic_class_template.mustache", "ext": ".py" } ] -enum_template_files = [ { "filename": "pydantic_enum_template.mustache", "ext": ".py" } ] def get_class_location(class_name, class_map, version): # Check if the current class has a parent class if class_map[class_name].superClass(): if class_map[class_name].superClass() in class_map: - return 'pydantic.' + version + "." + class_map[class_name].superClass() - elif class_map[class_name].superClass() == 'Base' or class_map[class_name].superClass() == None: + return "pydantic." + version + "." + class_map[class_name].superClass() + elif ( + class_map[class_name].superClass() == "Base" + or class_map[class_name].superClass() == None + ): return location(version) else: return location(version) + partials = {} + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_instances(text, render): instance = eval(render(text)) if "label" in instance: - value = instance['label'] + ' = "' + instance['label'] + '"' - if 'comment' in instance: - value += ' #' + instance['comment'] + value = instance["label"] + ' = "' + instance["label"] + '"' + if "comment" in instance: + value += " #" + instance["comment"] return value else: - return '' + return "" + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_imports(text, render): @@ -58,51 +66,58 @@ def _set_imports(text, render): try: res = eval(rendered) finally: - result = '' - classes = set () + result = "" + classes = set() if res: for val in res: if "range" in val: - classes.add(val['range'].split('#')[1]) + classes.add(val["range"].split("#")[1]) for val in classes: - result += "from ."+val+" import "+val+"\n" + result += "from ." + val + " import " + val + "\n" return result + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_default(text, render): attribute = eval(render(text)) if "range" in attribute and "isFixed" in attribute: - return ' = ' + attribute['range'].split('#')[1] + '.' + attribute['isFixed'] - if "multiplicity" in attribute and attribute['multiplicity'] in ['M:0..n'] and "range" in attribute: - return ' = None' + return " = " + attribute["range"].split("#")[1] + "." + attribute["isFixed"] + if ( + "multiplicity" in attribute + and attribute["multiplicity"] in ["M:0..n"] + and "range" in attribute + ): + return " = None" if "range" not in attribute and "isFixed" in attribute: - return ' = "' + attribute['isFixed'] + '"' - return '' + return ' = "' + attribute["isFixed"] + '"' + return "" + def _compute_data_type(attribute): - if 'label' in attribute and attribute['label'] == 'mRID': - return 'uuid.UUID' + if "label" in attribute and attribute["label"] == "mRID": + return "uuid.UUID" if "dataType" in attribute: - if attribute['dataType'].startswith('#'): - datatype = attribute['dataType'].split('#')[1] - if (datatype == 'Integer' or datatype == 'integer'): - return 'int' - if (datatype == 'Boolean'): - return 'bool' - if (datatype == 'String'): - return 'str' - if (datatype == 'DateTime'): - return 'str' - if (datatype == 'Date'): - return 'str' - if (datatype == 'String'): - return 'str' + if attribute["dataType"].startswith("#"): + datatype = attribute["dataType"].split("#")[1] + if datatype == "Integer" or datatype == "integer": + return "int" + if datatype == "Boolean": + return "bool" + if datatype == "String": + return "str" + if datatype == "DateTime": + return "str" + if datatype == "Date": + return "str" + if datatype == "String": + return "str" else: return "float" if "range" in attribute: - return attribute['range'].split('#')[1] + return attribute["range"].split("#")[1] + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_data_type(text, render): @@ -111,49 +126,55 @@ def _set_data_type(text, render): datatype = _compute_data_type(attribute) if "multiplicity" in attribute: - multiplicity = attribute['multiplicity'] - if multiplicity in ['M:1..1', '']: + multiplicity = attribute["multiplicity"] + if multiplicity in ["M:1..1", ""]: return datatype - if multiplicity in ['M:0..1']: - return 'Optional['+datatype+']' - elif multiplicity in ['M:0..n'] or 'M:0..' in multiplicity: - return 'Optional[List['+datatype+']]' - elif multiplicity in ['M:1', 'M:1..n'] or 'M:1..' in multiplicity: - return 'List['+datatype+']' + if multiplicity in ["M:0..1"]: + return "Optional[" + datatype + "]" + elif multiplicity in ["M:0..n"] or "M:0.." in multiplicity: + return "Optional[List[" + datatype + "]]" + elif multiplicity in ["M:1", "M:1..n"] or "M:1.." in multiplicity: + return "List[" + datatype + "]" else: - return 'List['+datatype+']' + return "List[" + datatype + "]" else: return datatype + def set_enum_classes(new_enum_classes): return + def set_float_classes(new_float_classes): return + def run_template(version_path, class_details): - if class_details['class_name'] in ['Float','Integer','String','Boolean','Date']: + if class_details["class_name"] in ["Float", "Integer", "String", "Boolean", "Date"]: templates = [] - elif class_details['has_instances'] == True: + elif class_details["has_instances"] == True: templates = enum_template_files else: templates = template_files - for template_info in templates: - class_file = os.path.join(version_path, class_details['class_name'] + template_info["ext"]) + class_file = os.path.join( + version_path, class_details["class_name"] + template_info["ext"] + ) if not os.path.exists(class_file): - with open(class_file, 'w') as file: - template_path = os.path.join(os.getcwd(), 'pydantic/templates', template_info["filename"]) - class_details['setDefault'] = _set_default - class_details['setDataType'] = _set_data_type - class_details['setImports'] = _set_imports - class_details['setInstances'] = _set_instances + with open(class_file, "w") as file: + template_path = os.path.join( + os.getcwd(), "pydantic/templates", template_info["filename"] + ) + class_details["setDefault"] = _set_default + class_details["setDataType"] = _set_data_type + class_details["setImports"] = _set_imports + class_details["setInstances"] = _set_instances with open(template_path) as f: args = { - 'data': class_details, - 'template': f, - 'partials_dict': partials + "data": class_details, + "template": f, + "partials_dict": partials, } output = chevron.render(**args) file.write(output) @@ -161,15 +182,17 @@ def run_template(version_path, class_details): def _create_init(path): init_file = path + "/__init__.py" - with open(init_file, 'w'): + with open(init_file, "w"): pass + # creates the Base class file, all classes inherit from this class def _copy_files(path): shutil.copy("pydantic/Base.py", path + "/Base.py") shutil.copy("pydantic/PositionPoint.py", path + "/PositionPoint.py") shutil.copy("pydantic/util.py", path + "/util.py") + def resolve_headers(path): filenames = glob.glob(path + "/*.py") include_names = [] @@ -177,5 +200,14 @@ def resolve_headers(path): include_names.append(os.path.splitext(os.path.basename(filename))[0]) with open(path + "/__init__.py", "w") as header_file: for include_name in include_names: - header_file.write("from " + "." + include_name + " import " + include_name + " as " + include_name + "\n") + header_file.write( + "from " + + "." + + include_name + + " import " + + include_name + + " as " + + include_name + + "\n" + ) header_file.close() diff --git a/pydantic/util.py b/pydantic/util.py index e29622f8..d10c0f01 100644 --- a/pydantic/util.py +++ b/pydantic/util.py @@ -4,6 +4,7 @@ ) from typing import Iterator + def is_recursion_validation_error(exc: ValidationError) -> bool: errors = exc.errors() return len(errors) == 1 and errors[0]["type"] == "recursion_loop" From b3d7f6cff38027e6f7818bb03acd1cda0a42ac33 Mon Sep 17 00:00:00 2001 From: chicco785 Date: Sat, 29 Jul 2023 06:32:52 +0000 Subject: [PATCH 04/34] docs(release_notes): update RELEASE_NOTES.md --- RELEASE_NOTES.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c4762c44..2d42dc63 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,10 @@ # cimgen Release Notes -## 0.0.1-dev - 2023-07-28 +## 0.0.1-dev - 2023-07-29 + +### Features + +- Support pydantic code generation (PR #5 by @chicco785) ### Continuous Integration From 2d2faeb46529f35e737687c734c2f00611fa0da4 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 12:49:02 +0200 Subject: [PATCH 05/34] support cyclic reference serialization --- pydantic/langPack.py | 22 +++++++++++++++++++ .../pydantic_class_template.mustache | 8 ++++++- pydantic/util.py | 21 ++++++++++++++++-- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index e51f1a2d..6d5cd092 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -94,6 +94,12 @@ def _set_default(text, render): return "" +def _is_primitive(datatype): + if datatype in ['str','int','bool','float']: + return True + else: + return False + def _compute_data_type(attribute): if "label" in attribute and attribute["label"] == "mRID": return "uuid.UUID" @@ -141,6 +147,21 @@ def _set_data_type(text, render): return datatype +# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) +def _set_validator(text, render): + attribute = eval(render(text)) + + datatype = _compute_data_type(attribute) + + if "multiplicity" in attribute and not _is_primitive(datatype): + multiplicity = attribute["multiplicity"] + if (multiplicity in ["M:0..n"] or "M:0.." in multiplicity) or (multiplicity in ["M:1", "M:1..n"] or "M:1.." in multiplicity): + return 'val_' + datatype + '_wrap = field_validator("'+ datatype + '", mode="wrap")(cyclic_references_validator)' + else: + return "" + else: + return "" + def set_enum_classes(new_enum_classes): return @@ -170,6 +191,7 @@ def run_template(version_path, class_details): class_details["setDataType"] = _set_data_type class_details["setImports"] = _set_imports class_details["setInstances"] = _set_instances + class_details["setValidator"] = _set_validator with open(template_path) as f: args = { "data": class_details, diff --git a/pydantic/templates/pydantic_class_template.mustache b/pydantic/templates/pydantic_class_template.mustache index 917b7064..2f68dfbb 100644 --- a/pydantic/templates/pydantic_class_template.mustache +++ b/pydantic/templates/pydantic_class_template.mustache @@ -4,9 +4,11 @@ from .Base import CgmesProfileEnum import uuid from pydantic import ( ConfigDict, - Field + Field, + field_validator ) from typing import Optional, Iterator, List +from .util import cyclic_references_validator {{#setImports}}{{attributes}}{{/setImports}} class {{class_name}}({{sub_class_of}}): @@ -26,4 +28,8 @@ class {{class_name}}({{sub_class_of}}): {{label}}: {{#setDataType}}{{.}}{{/setDataType}}{{#setDefault}}{{.}}{{/setDefault}} {{/attributes}} + {{#attributes}} + {{#setValidator}}{{.}}{{/setValidator}} + {{/attributes}} + model_config = ConfigDict(from_attributes=True) diff --git a/pydantic/util.py b/pydantic/util.py index d10c0f01..e183ab27 100644 --- a/pydantic/util.py +++ b/pydantic/util.py @@ -1,8 +1,10 @@ from contextlib import contextmanager from pydantic import ( - ValidationError, +ValidationError, + ValidationInfo, + ValidatorFunctionWrapHandler, ) -from typing import Iterator +from typing import Iterator, List def is_recursion_validation_error(exc: ValidationError) -> bool: @@ -17,3 +19,18 @@ def suppress_recursion_validation_error() -> Iterator[None]: except ValidationError as exc: if not is_recursion_validation_error(exc): raise exc + +def cyclic_references_validator( + v: List, handler: ValidatorFunctionWrapHandler, info: ValidationInfo +): + try: + return handler(v) + except ValidationError as exc: + if not (is_recursion_validation_error(exc) and isinstance(v, list)): + raise exc + + value_without_cyclic_refs = [] + for child in v: + with suppress_recursion_validation_error(): + value_without_cyclic_refs.extend(handler([child])) + return handler(value_without_cyclic_refs) From b2779cce6ae0f2c1cca8f32327219c412b238851 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 12:53:23 +0200 Subject: [PATCH 06/34] add pydantic to tests --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c366ac5..dc0f8f46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: - cpp - java - javascript + - pydantic steps: - uses: actions/checkout@v3 From 03a514d76a552cc766929bc600b9f81a84378f58 Mon Sep 17 00:00:00 2001 From: Bot Date: Mon, 31 Jul 2023 10:49:39 +0000 Subject: [PATCH 07/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/langPack.py | 16 +++++++++++++--- pydantic/util.py | 3 ++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 6d5cd092..00d8842d 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -95,11 +95,12 @@ def _set_default(text, render): def _is_primitive(datatype): - if datatype in ['str','int','bool','float']: + if datatype in ["str", "int", "bool", "float"]: return True else: return False + def _compute_data_type(attribute): if "label" in attribute and attribute["label"] == "mRID": return "uuid.UUID" @@ -155,13 +156,22 @@ def _set_validator(text, render): if "multiplicity" in attribute and not _is_primitive(datatype): multiplicity = attribute["multiplicity"] - if (multiplicity in ["M:0..n"] or "M:0.." in multiplicity) or (multiplicity in ["M:1", "M:1..n"] or "M:1.." in multiplicity): - return 'val_' + datatype + '_wrap = field_validator("'+ datatype + '", mode="wrap")(cyclic_references_validator)' + if (multiplicity in ["M:0..n"] or "M:0.." in multiplicity) or ( + multiplicity in ["M:1", "M:1..n"] or "M:1.." in multiplicity + ): + return ( + "val_" + + datatype + + '_wrap = field_validator("' + + datatype + + '", mode="wrap")(cyclic_references_validator)' + ) else: return "" else: return "" + def set_enum_classes(new_enum_classes): return diff --git a/pydantic/util.py b/pydantic/util.py index e183ab27..edc71da3 100644 --- a/pydantic/util.py +++ b/pydantic/util.py @@ -1,6 +1,6 @@ from contextlib import contextmanager from pydantic import ( -ValidationError, + ValidationError, ValidationInfo, ValidatorFunctionWrapHandler, ) @@ -20,6 +20,7 @@ def suppress_recursion_validation_error() -> Iterator[None]: if not is_recursion_validation_error(exc): raise exc + def cyclic_references_validator( v: List, handler: ValidatorFunctionWrapHandler, info: ValidationInfo ): From 267849c4198eccc74800e07848467ad13dde0c8c Mon Sep 17 00:00:00 2001 From: chicco785 Date: Mon, 31 Jul 2023 10:50:32 +0000 Subject: [PATCH 08/34] docs(release_notes): update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2d42dc63..4d41dbb7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # cimgen Release Notes -## 0.0.1-dev - 2023-07-29 +## 0.0.1-dev - 2023-07-31 ### Features From de05e72bfa5b458fbcf227fed7d6fb48e54b7e5d Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 12:55:11 +0200 Subject: [PATCH 09/34] fix import issue --- pydantic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 9f57f01a..5809fef4 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -1 +1 @@ -import pypdantic.langPack +import pydantic.langPack From d6ac0fba2b3d2c561da351b39e20fa37ba14f9ab Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 13:02:04 +0200 Subject: [PATCH 10/34] improve datatype support --- pydantic/langPack.py | 10 +++++++--- pydantic/templates/pydantic_class_template.mustache | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 00d8842d..c9d5a523 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -95,7 +95,7 @@ def _set_default(text, render): def _is_primitive(datatype): - if datatype in ["str", "int", "bool", "float"]: + if datatype in ["str", "int", "bool", "float", "date", "time", "datetime"]: return True else: return False @@ -115,9 +115,13 @@ def _compute_data_type(attribute): if datatype == "String": return "str" if datatype == "DateTime": - return "str" + return "datetime" if datatype == "Date": - return "str" + return "date" + if datatype == "Time": + return "time" + if datatype == "Float": + return "float" if datatype == "String": return "str" else: diff --git a/pydantic/templates/pydantic_class_template.mustache b/pydantic/templates/pydantic_class_template.mustache index 2f68dfbb..14f7c9fa 100644 --- a/pydantic/templates/pydantic_class_template.mustache +++ b/pydantic/templates/pydantic_class_template.mustache @@ -7,6 +7,7 @@ from pydantic import ( Field, field_validator ) +from datetime import date, datetime, time from typing import Optional, Iterator, List from .util import cyclic_references_validator {{#setImports}}{{attributes}}{{/setImports}} From ab3a612780c80721f7754b0e1cfa1db567d09517 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 13:20:03 +0200 Subject: [PATCH 11/34] fix multiplicity support in validator generation --- pydantic/langPack.py | 4 ++-- pydantic/templates/pydantic_class_template.mustache | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index c9d5a523..3ffe40f5 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -160,8 +160,8 @@ def _set_validator(text, render): if "multiplicity" in attribute and not _is_primitive(datatype): multiplicity = attribute["multiplicity"] - if (multiplicity in ["M:0..n"] or "M:0.." in multiplicity) or ( - multiplicity in ["M:1", "M:1..n"] or "M:1.." in multiplicity + if (multiplicity in ["M:0..n"] or ("M:0.." in multiplicity and "M:0..1" not in multiplicity)) or ( + multiplicity in ["M:1", "M:1..n"] or ("M:1.." in multiplicity and "M:1..1" not in multiplicity) ): return ( "val_" diff --git a/pydantic/templates/pydantic_class_template.mustache b/pydantic/templates/pydantic_class_template.mustache index 14f7c9fa..0acd2bfe 100644 --- a/pydantic/templates/pydantic_class_template.mustache +++ b/pydantic/templates/pydantic_class_template.mustache @@ -28,7 +28,7 @@ class {{class_name}}({{sub_class_of}}): {{#attributes}} {{label}}: {{#setDataType}}{{.}}{{/setDataType}}{{#setDefault}}{{.}}{{/setDefault}} {{/attributes}} - + {{#attributes}} {{#setValidator}}{{.}}{{/setValidator}} {{/attributes}} From bfcef79b9222aefe11ef090d6c93727f49edf921 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 13:44:13 +0200 Subject: [PATCH 12/34] add schema and pydantic to docker build --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d7f03e7d..f0d99a50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,9 @@ COPY cpp/ /CIMgen/cpp/ COPY java/ /CIMgen/java/ COPY javascript/ /CIMgen/javascript/ COPY python/ /CIMgen/python/ +COPY pydantic/ /CIMgen/pydantic/ COPY CIMgen.py build.py /CIMgen/ +COPY cgmes_schema/ /cgmes_schema WORKDIR /CIMgen ENTRYPOINT [ "/usr/bin/python3", "build.py", "--outdir=/cgmes_output", "--schemadir=/cgmes_schema" ] CMD [ "--langdir=cpp" ] From 65aa21161ea5a8776cf97f85f9b16ba6af76f876 Mon Sep 17 00:00:00 2001 From: Bot Date: Mon, 31 Jul 2023 11:20:44 +0000 Subject: [PATCH 13/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/langPack.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 3ffe40f5..a782b92b 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -160,8 +160,12 @@ def _set_validator(text, render): if "multiplicity" in attribute and not _is_primitive(datatype): multiplicity = attribute["multiplicity"] - if (multiplicity in ["M:0..n"] or ("M:0.." in multiplicity and "M:0..1" not in multiplicity)) or ( - multiplicity in ["M:1", "M:1..n"] or ("M:1.." in multiplicity and "M:1..1" not in multiplicity) + if ( + multiplicity in ["M:0..n"] + or ("M:0.." in multiplicity and "M:0..1" not in multiplicity) + ) or ( + multiplicity in ["M:1", "M:1..n"] + or ("M:1.." in multiplicity and "M:1..1" not in multiplicity) ): return ( "val_" From d7021fd2964425bf41bdd48137f6c88b8fc1b1ca Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Mon, 31 Jul 2023 15:16:57 +0200 Subject: [PATCH 14/34] fix Enum for CgmesProfile --- pydantic/Base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic/Base.py b/pydantic/Base.py index 37c754a2..bb4ab2ec 100644 --- a/pydantic/Base.py +++ b/pydantic/Base.py @@ -10,8 +10,8 @@ class CgmesProfileEnum(IntEnum): DY = 4 GL = 5 DL = 5 - TP_DB = 7 - ED_BD = 8 + TP_BD = 7 + EQ_BD = 8 class Base(BaseModel): From 169f3bf343af4de68f2a397a7ce0c018c2dc4c9b Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 12:13:30 +0200 Subject: [PATCH 15/34] generates only EQ profile (tested in CIM-Manager) --- CIMgen.py | 28 ++- pydantic/Base.py | 18 +- pydantic/enum_header.py | 14 ++ pydantic/langPack.py | 198 ++++++++++-------- .../{PositionPoint.py => schema_header.py} | 40 ++-- .../pydantic_class_template.mustache | 17 +- .../templates/pydantic_enum_template.mustache | 2 - pydantic/util.py | 4 +- 8 files changed, 171 insertions(+), 150 deletions(-) create mode 100644 pydantic/enum_header.py rename pydantic/{PositionPoint.py => schema_header.py} (80%) diff --git a/CIMgen.py b/CIMgen.py index 8b221937..7b9d8e38 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -529,10 +529,6 @@ def _write_files(class_details, outputPath, version): class_details["class_location"] = class_details["langPack"].base[ "class_location" ](version) - class_details["super_init"] = False - else: - # If class is a subclass a super().__init__() is needed - class_details["super_init"] = True # The entry dataType for an attribute is only set for basic data types. If the entry is not set here, the attribute # is a reference to another class and therefore the entry dataType is generated and set to the multiplicity @@ -686,6 +682,14 @@ def addSubClassesOfSubClasses(class_dict): recursivelyAddSubClasses(class_dict, className) ) +def addSubClassesOfSubClassesClean(class_dict, source): + temp = {} + for className in class_dict: + for name in class_dict[className].subClasses(): + if name not in class_dict: + temp[name] = source[name] + addSubClassesOfSubClassesClean(temp, source) + class_dict.update(temp) def cim_generate(directory, outputPath, version, langPack): """Generates cgmes python classes from cgmes ontology @@ -729,6 +733,8 @@ def cim_generate(directory, outputPath, version, langPack): # merge classes from different profiles into one class and track origin of the classes and their attributes class_dict_with_origins = _merge_classes(profiles_dict) + clean_class_dict = {} + # work out the subclasses for each class by noting the reverse relationship for className in class_dict_with_origins: superClassName = class_dict_with_origins[className].superClass() @@ -742,7 +748,19 @@ def cim_generate(directory, outputPath, version, langPack): # recursively add the subclasses of subclasses addSubClassesOfSubClasses(class_dict_with_origins) + for className in class_dict_with_origins: + superClassName = class_dict_with_origins[className].superClass() + if superClassName == None and class_dict_with_origins[className].has_instances(): + clean_class_dict[className] = class_dict_with_origins[className] + + for className in class_dict_with_origins: + superClassName = class_dict_with_origins[className].superClass() + if superClassName == None and not class_dict_with_origins[className].has_instances(): + clean_class_dict[className] = class_dict_with_origins[className] + + addSubClassesOfSubClassesClean(clean_class_dict, class_dict_with_origins) + # get information for writing python files and write python files - _write_python_files(class_dict_with_origins, langPack, outputPath, version) + _write_python_files(clean_class_dict, langPack, outputPath, version) logger.info("Elapsed Time: {}s\n\n".format(time() - t0)) diff --git a/pydantic/Base.py b/pydantic/Base.py index bb4ab2ec..60854b64 100644 --- a/pydantic/Base.py +++ b/pydantic/Base.py @@ -1,24 +1,12 @@ from pydantic import BaseModel -from enum import IntEnum - - -class CgmesProfileEnum(IntEnum): - EQ = 0 - SSH = 1 - TP = 2 - SV = 3 - DY = 4 - GL = 5 - DL = 5 - TP_BD = 7 - EQ_BD = 8 - class Base(BaseModel): """ Base Class for CIM """ + """ + not valid for pydantic 2.0 class Config: @staticmethod def schema_extra(schema: dict, _): @@ -26,7 +14,7 @@ def schema_extra(schema: dict, _): for k, v in schema.get("properties", {}).items(): if not v.get("hidden", False): props[k] = v - schema["properties"] = props + schema["properties"] = props """ def printxml(self, dict={}): return dict diff --git a/pydantic/enum_header.py b/pydantic/enum_header.py new file mode 100644 index 00000000..1e20ba80 --- /dev/null +++ b/pydantic/enum_header.py @@ -0,0 +1,14 @@ +from enum import Enum, IntEnum + + +class CgmesProfileEnum(IntEnum): + EQ = 0 + SSH = 1 + TP = 2 + SV = 3 + DY = 4 + GL = 5 + DL = 5 + TP_BD = 7 + EQ_BD = 8 + diff --git a/pydantic/langPack.py b/pydantic/langPack.py index a782b92b..6aff18e0 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -29,6 +29,7 @@ def location(version): template_files = [{"filename": "pydantic_class_template.mustache", "ext": ".py"}] enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] +required_profiles = ['EQ'] def get_class_location(class_name, class_map, version): # Check if the current class has a parent class @@ -60,38 +61,31 @@ def _set_instances(text, render): # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) -def _set_imports(text, render): - rendered = render(text) - res = None - try: - res = eval(rendered) - finally: - result = "" - classes = set() - if res: - for val in res: - if "range" in val: - classes.add(val["range"].split("#")[1]) - for val in classes: - result += "from ." + val + " import " + val + "\n" - return result - - -# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) -def _set_default(text, render): +def _set_attribute(text, render): attribute = eval(render(text)) + if is_required_profile(attribute['attr_origin']): + return attribute['label'] + ":" + _set_data_type(attribute) + _set_default(attribute) + else: + return "" + +def _set_default(attribute): if "range" in attribute and "isFixed" in attribute: return " = " + attribute["range"].split("#")[1] + "." + attribute["isFixed"] - if ( - "multiplicity" in attribute - and attribute["multiplicity"] in ["M:0..n"] - and "range" in attribute - ): - return " = None" - if "range" not in attribute and "isFixed" in attribute: - return ' = "' + attribute["isFixed"] + '"' - return "" + elif "multiplicity" in attribute: + multiplicity = attribute["multiplicity"] + if multiplicity in ["M:1", "M:1..1"]: + return "" + if multiplicity in ["M:0..1"]: + return "" + elif multiplicity in ["M:0..n"] or "M:0.." in multiplicity: + return "" + elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: + return "" + else: + return "" + else: + return "" def _is_primitive(datatype): @@ -116,8 +110,10 @@ def _compute_data_type(attribute): return "str" if datatype == "DateTime": return "datetime" + if datatype == "MonthDay": + return "str" #TO BE FIXED if datatype == "Date": - return "date" + return "str" #TO BE FIXED if datatype == "Time": return "time" if datatype == "Float": @@ -127,24 +123,22 @@ def _compute_data_type(attribute): else: return "float" if "range" in attribute: + #return "'"+attribute["range"].split("#")[1]+"'" return attribute["range"].split("#")[1] - -# called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) -def _set_data_type(text, render): - attribute = eval(render(text)) +def _set_data_type(attribute): datatype = _compute_data_type(attribute) if "multiplicity" in attribute: multiplicity = attribute["multiplicity"] - if multiplicity in ["M:1..1", ""]: + if multiplicity in ["M:1", "M:1..1", ""]: return datatype if multiplicity in ["M:0..1"]: return "Optional[" + datatype + "]" elif multiplicity in ["M:0..n"] or "M:0.." in multiplicity: return "Optional[List[" + datatype + "]]" - elif multiplicity in ["M:1", "M:1..n"] or "M:1.." in multiplicity: + elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: return "List[" + datatype + "]" else: return "List[" + datatype + "]" @@ -158,24 +152,14 @@ def _set_validator(text, render): datatype = _compute_data_type(attribute) - if "multiplicity" in attribute and not _is_primitive(datatype): - multiplicity = attribute["multiplicity"] - if ( - multiplicity in ["M:0..n"] - or ("M:0.." in multiplicity and "M:0..1" not in multiplicity) - ) or ( - multiplicity in ["M:1", "M:1..n"] - or ("M:1.." in multiplicity and "M:1..1" not in multiplicity) - ): - return ( - "val_" - + datatype - + '_wrap = field_validator("' - + datatype - + '", mode="wrap")(cyclic_references_validator)' - ) - else: - return "" + if not _is_primitive(datatype) and is_required_profile(attribute['attr_origin']): + return ( + "val_" + + attribute['label'] + + '_wrap = field_validator("' + + attribute['label'] + + '", mode="wrap")(cyclic_references_validator)' + ) else: return "" @@ -188,36 +172,81 @@ def set_float_classes(new_float_classes): return +def has_unit_attribute(attributes): + for attr in attributes: + if attr['label'] == 'unit': + return True + return False + + +def is_required_profile(class_origin): + for origin in class_origin: + if origin['origin'] in required_profiles: + return True + return False + def run_template(version_path, class_details): - if class_details["class_name"] in ["Float", "Integer", "String", "Boolean", "Date"]: - templates = [] + if (class_details["class_name"] in ["Float", "Integer", "String", "Boolean", "Date", "DateTime", "MonthDay", "PositionPoint", "Decimal"]) or class_details["is_a_float"] == True or "Version" in class_details["class_name"] or has_unit_attribute(class_details["attributes"]) or not is_required_profile(class_details['class_origin']): + return elif class_details["has_instances"] == True: - templates = enum_template_files + run_template_enum(version_path, class_details, enum_template_files) else: - templates = template_files + run_template_schema(version_path, class_details, template_files) + +def run_template_enum(version_path, class_details, templates): for template_info in templates: class_file = os.path.join( - version_path, class_details["class_name"] + template_info["ext"] + version_path, "enum" + template_info["ext"] ) if not os.path.exists(class_file): with open(class_file, "w") as file: - template_path = os.path.join( - os.getcwd(), "pydantic/templates", template_info["filename"] + header_file_path = os.path.join( + os.getcwd(), "pydantic", "enum_header.py" ) - class_details["setDefault"] = _set_default - class_details["setDataType"] = _set_data_type - class_details["setImports"] = _set_imports - class_details["setInstances"] = _set_instances - class_details["setValidator"] = _set_validator - with open(template_path) as f: - args = { - "data": class_details, - "template": f, - "partials_dict": partials, - } - output = chevron.render(**args) - file.write(output) + header_file = open(header_file_path, 'r') + file.write(header_file.read()) + with open(class_file, "a") as file: + template_path = os.path.join( + os.getcwd(), "pydantic/templates", template_info["filename"] + ) + class_details["setInstances"] = _set_instances + with open(template_path) as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) + + +def run_template_schema(version_path, class_details, templates): + for template_info in templates: + class_file = os.path.join( + version_path, "schema" + template_info["ext"] + ) + if not os.path.exists(class_file): + with open(class_file, "w") as file: + schema_file_path = os.path.join( + os.getcwd(), "pydantic", "schema_header.py" + ) + schema_file = open(schema_file_path, 'r') + file.write(schema_file.read()) + with open(class_file, "a") as file: + template_path = os.path.join( + os.getcwd(), "pydantic/templates", template_info["filename"] + ) + class_details["setAttribute"] = _set_attribute + class_details["setValidator"] = _set_validator + with open(template_path) as f: + args = { + "data": class_details, + "template": f, + "partials_dict": partials, + } + output = chevron.render(**args) + file.write(output) def _create_init(path): @@ -228,26 +257,9 @@ def _create_init(path): # creates the Base class file, all classes inherit from this class def _copy_files(path): - shutil.copy("pydantic/Base.py", path + "/Base.py") - shutil.copy("pydantic/PositionPoint.py", path + "/PositionPoint.py") - shutil.copy("pydantic/util.py", path + "/util.py") + shutil.copy("./pydantic/Base.py", path + "/Base.py") + shutil.copy("./pydantic/util.py", path + "/util.py") def resolve_headers(path): - filenames = glob.glob(path + "/*.py") - include_names = [] - for filename in filenames: - include_names.append(os.path.splitext(os.path.basename(filename))[0]) - with open(path + "/__init__.py", "w") as header_file: - for include_name in include_names: - header_file.write( - "from " - + "." - + include_name - + " import " - + include_name - + " as " - + include_name - + "\n" - ) - header_file.close() + pass diff --git a/pydantic/PositionPoint.py b/pydantic/schema_header.py similarity index 80% rename from pydantic/PositionPoint.py rename to pydantic/schema_header.py index c6c61e59..d19e05b7 100644 --- a/pydantic/PositionPoint.py +++ b/pydantic/schema_header.py @@ -1,19 +1,23 @@ from __future__ import annotations -from .Base import Base, CgmesProfileEnum - -from pydantic import ConfigDict, computed_field, field_validator, Field - -from geoalchemy2.shape import to_shape -from geoalchemy2.elements import WKBElement -from typing import Optional, List - -from shapely.geometry import Point -from geoalchemy2.shape import to_shape -from .Location import Location - +import uuid +from pydantic import ( + ConfigDict, + Field, + field_validator, +# computed_field +) +#from geoalchemy2.shape import to_shape +#from geoalchemy2.elements import WKBElement +#from shapely.geometry import Point +from datetime import date, datetime, time +from typing import Optional, Iterator, List +from .Base import Base +from .util import cyclic_references_validator +from .enum import * +""" class PositionPoint(Base): - """ + Set of spatial coordinates that determine a point, defined in the coordinate system specified in 'Location.CoordinateSystem'. Use a single position point instance to desribe a point-oriented location. Use a sequence of position points to describe a line-oriented object (physical location of non-point oriented objects like cables or lines), or area of an object (like a substation or a geographical zone - in this case, have first and last position point with the same values). :Location: Location described by this position point. @@ -21,7 +25,7 @@ class PositionPoint(Base): :xPosition: X axis position. :yPosition: Y axis position. :zPosition: (if applicable) Z axis position. - """ + possibleProfileList: dict = Field( default={ @@ -43,13 +47,12 @@ class PositionPoint(Base): "zPosition": [ CgmesProfileEnum.GL, ], - }, - hidden=True, + } ) - Location: List[Location] + Location: List['Location'] sequenceNumber: Optional[int] - point: Point # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + point: Point #= Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database model_config = ConfigDict(from_attributes=True) @@ -84,3 +87,4 @@ def validate_point_format(cls, v): return Point(point) else: raise ValueError("must be a Point or a WKBElement") +""" \ No newline at end of file diff --git a/pydantic/templates/pydantic_class_template.mustache b/pydantic/templates/pydantic_class_template.mustache index 0acd2bfe..095e8363 100644 --- a/pydantic/templates/pydantic_class_template.mustache +++ b/pydantic/templates/pydantic_class_template.mustache @@ -1,16 +1,3 @@ -from __future__ import annotations -from .{{sub_class_of}} import {{sub_class_of}} -from .Base import CgmesProfileEnum -import uuid -from pydantic import ( - ConfigDict, - Field, - field_validator -) -from datetime import date, datetime, time -from typing import Optional, Iterator, List -from .util import cyclic_references_validator -{{#setImports}}{{attributes}}{{/setImports}} class {{class_name}}({{sub_class_of}}): ''' @@ -23,10 +10,10 @@ class {{class_name}}({{sub_class_of}}): possibleProfileList: dict = Field(default={'class': [{{#class_origin}}CgmesProfileEnum.{{origin}}, {{/class_origin}}], {{#attributes}}'{{label}}': [{{#attr_origin}}CgmesProfileEnum.{{origin}}, {{/attr_origin}}], - {{/attributes}} }, hidden=True) + {{/attributes}} }) {{#attributes}} - {{label}}: {{#setDataType}}{{.}}{{/setDataType}}{{#setDefault}}{{.}}{{/setDefault}} + {{#setAttribute}}{{.}}{{/setAttribute}} {{/attributes}} {{#attributes}} diff --git a/pydantic/templates/pydantic_enum_template.mustache b/pydantic/templates/pydantic_enum_template.mustache index f48e9cca..4f5aa94f 100644 --- a/pydantic/templates/pydantic_enum_template.mustache +++ b/pydantic/templates/pydantic_enum_template.mustache @@ -1,5 +1,3 @@ -from .{{sub_class_of}} import {{sub_class_of}} -from enum import Enum class {{class_name}}(str,Enum): diff --git a/pydantic/util.py b/pydantic/util.py index edc71da3..d08616f5 100644 --- a/pydantic/util.py +++ b/pydantic/util.py @@ -4,7 +4,7 @@ ValidationInfo, ValidatorFunctionWrapHandler, ) -from typing import Iterator, List +from typing import Iterator, Any def is_recursion_validation_error(exc: ValidationError) -> bool: @@ -22,7 +22,7 @@ def suppress_recursion_validation_error() -> Iterator[None]: def cyclic_references_validator( - v: List, handler: ValidatorFunctionWrapHandler, info: ValidationInfo + v: Any, handler: ValidatorFunctionWrapHandler, info: ValidationInfo ): try: return handler(v) From 860ab89a99ed988ea94c8d013fea62e989514f30 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 12:33:54 +0200 Subject: [PATCH 16/34] fix rendering for usage of "e; in CMGES3.0 --- pydantic/langPack.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 6aff18e0..f81e5e6f 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -50,15 +50,20 @@ def get_class_location(class_name, class_map, version): # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_instances(text, render): - instance = eval(render(text)) + instance = None + try: + instance = eval(render(text)) + except: + rendered = render(text) + rendered = rendered.replace('"','"') + instance = eval(rendered) if "label" in instance: value = instance["label"] + ' = "' + instance["label"] + '"' if "comment" in instance: value += " #" + instance["comment"] return value else: - return "" - + return "" # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_attribute(text, render): From 91f53383bf84661096917a3c6d02a93a4a4355a3 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 12:40:28 +0200 Subject: [PATCH 17/34] fix copy file path --- pydantic/langPack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index f81e5e6f..660e0d63 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -262,8 +262,10 @@ def _create_init(path): # creates the Base class file, all classes inherit from this class def _copy_files(path): - shutil.copy("./pydantic/Base.py", path + "/Base.py") - shutil.copy("./pydantic/util.py", path + "/util.py") + shutil.copy(os.path.join( + os.getcwd(), "pydantic/Base.py"), path + "/Base.py") + shutil.copy(os.path.join( + os.getcwd(), "pydantic/util.py"), path + "/util.py") def resolve_headers(path): From d9cdc71fb8c3dceb0ee3c9bfbfbac17dad09782c Mon Sep 17 00:00:00 2001 From: chicco785 Date: Wed, 2 Aug 2023 10:14:14 +0000 Subject: [PATCH 18/34] docs(release_notes): update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4d41dbb7..0e5ca49a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # cimgen Release Notes -## 0.0.1-dev - 2023-07-31 +## 0.0.1-dev - 2023-08-02 ### Features From 35d00f0eeaaf6d610ff806cd554aaa5c32e32c83 Mon Sep 17 00:00:00 2001 From: Bot Date: Wed, 2 Aug 2023 10:14:57 +0000 Subject: [PATCH 19/34] Automated Black fmt fixes Signed-off-by: Bot --- CIMgen.py | 14 +++++++-- pydantic/Base.py | 1 + pydantic/enum_header.py | 1 - pydantic/langPack.py | 65 ++++++++++++++++++++++++++------------- pydantic/schema_header.py | 11 ++++--- 5 files changed, 62 insertions(+), 30 deletions(-) diff --git a/CIMgen.py b/CIMgen.py index 7b9d8e38..30821683 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -682,6 +682,7 @@ def addSubClassesOfSubClasses(class_dict): recursivelyAddSubClasses(class_dict, className) ) + def addSubClassesOfSubClassesClean(class_dict, source): temp = {} for className in class_dict: @@ -691,6 +692,7 @@ def addSubClassesOfSubClassesClean(class_dict, source): addSubClassesOfSubClassesClean(temp, source) class_dict.update(temp) + def cim_generate(directory, outputPath, version, langPack): """Generates cgmes python classes from cgmes ontology @@ -750,12 +752,18 @@ def cim_generate(directory, outputPath, version, langPack): for className in class_dict_with_origins: superClassName = class_dict_with_origins[className].superClass() - if superClassName == None and class_dict_with_origins[className].has_instances(): + if ( + superClassName == None + and class_dict_with_origins[className].has_instances() + ): clean_class_dict[className] = class_dict_with_origins[className] - + for className in class_dict_with_origins: superClassName = class_dict_with_origins[className].superClass() - if superClassName == None and not class_dict_with_origins[className].has_instances(): + if ( + superClassName == None + and not class_dict_with_origins[className].has_instances() + ): clean_class_dict[className] = class_dict_with_origins[className] addSubClassesOfSubClassesClean(clean_class_dict, class_dict_with_origins) diff --git a/pydantic/Base.py b/pydantic/Base.py index 60854b64..fe9111b4 100644 --- a/pydantic/Base.py +++ b/pydantic/Base.py @@ -1,5 +1,6 @@ from pydantic import BaseModel + class Base(BaseModel): """ Base Class for CIM diff --git a/pydantic/enum_header.py b/pydantic/enum_header.py index 1e20ba80..d15836fd 100644 --- a/pydantic/enum_header.py +++ b/pydantic/enum_header.py @@ -11,4 +11,3 @@ class CgmesProfileEnum(IntEnum): DL = 5 TP_BD = 7 EQ_BD = 8 - diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 660e0d63..79559124 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -29,7 +29,8 @@ def location(version): template_files = [{"filename": "pydantic_class_template.mustache", "ext": ".py"}] enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] -required_profiles = ['EQ'] +required_profiles = ["EQ"] + def get_class_location(class_name, class_map, version): # Check if the current class has a parent class @@ -69,11 +70,17 @@ def _set_instances(text, render): def _set_attribute(text, render): attribute = eval(render(text)) - if is_required_profile(attribute['attr_origin']): - return attribute['label'] + ":" + _set_data_type(attribute) + _set_default(attribute) + if is_required_profile(attribute["attr_origin"]): + return ( + attribute["label"] + + ":" + + _set_data_type(attribute) + + _set_default(attribute) + ) else: return "" + def _set_default(attribute): if "range" in attribute and "isFixed" in attribute: return " = " + attribute["range"].split("#")[1] + "." + attribute["isFixed"] @@ -116,9 +123,9 @@ def _compute_data_type(attribute): if datatype == "DateTime": return "datetime" if datatype == "MonthDay": - return "str" #TO BE FIXED + return "str" # TO BE FIXED if datatype == "Date": - return "str" #TO BE FIXED + return "str" # TO BE FIXED if datatype == "Time": return "time" if datatype == "Float": @@ -128,11 +135,11 @@ def _compute_data_type(attribute): else: return "float" if "range" in attribute: - #return "'"+attribute["range"].split("#")[1]+"'" + # return "'"+attribute["range"].split("#")[1]+"'" return attribute["range"].split("#")[1] -def _set_data_type(attribute): +def _set_data_type(attribute): datatype = _compute_data_type(attribute) if "multiplicity" in attribute: @@ -157,12 +164,12 @@ def _set_validator(text, render): datatype = _compute_data_type(attribute) - if not _is_primitive(datatype) and is_required_profile(attribute['attr_origin']): + if not _is_primitive(datatype) and is_required_profile(attribute["attr_origin"]): return ( "val_" - + attribute['label'] + + attribute["label"] + '_wrap = field_validator("' - + attribute['label'] + + attribute["label"] + '", mode="wrap")(cyclic_references_validator)' ) else: @@ -179,19 +186,39 @@ def set_float_classes(new_float_classes): def has_unit_attribute(attributes): for attr in attributes: - if attr['label'] == 'unit': + if attr["label"] == "unit": return True return False def is_required_profile(class_origin): for origin in class_origin: - if origin['origin'] in required_profiles: + if origin["origin"] in required_profiles: return True return False + def run_template(version_path, class_details): - if (class_details["class_name"] in ["Float", "Integer", "String", "Boolean", "Date", "DateTime", "MonthDay", "PositionPoint", "Decimal"]) or class_details["is_a_float"] == True or "Version" in class_details["class_name"] or has_unit_attribute(class_details["attributes"]) or not is_required_profile(class_details['class_origin']): + if ( + ( + class_details["class_name"] + in [ + "Float", + "Integer", + "String", + "Boolean", + "Date", + "DateTime", + "MonthDay", + "PositionPoint", + "Decimal", + ] + ) + or class_details["is_a_float"] == True + or "Version" in class_details["class_name"] + or has_unit_attribute(class_details["attributes"]) + or not is_required_profile(class_details["class_origin"]) + ): return elif class_details["has_instances"] == True: run_template_enum(version_path, class_details, enum_template_files) @@ -201,15 +228,13 @@ def run_template(version_path, class_details): def run_template_enum(version_path, class_details, templates): for template_info in templates: - class_file = os.path.join( - version_path, "enum" + template_info["ext"] - ) + class_file = os.path.join(version_path, "enum" + template_info["ext"]) if not os.path.exists(class_file): with open(class_file, "w") as file: header_file_path = os.path.join( os.getcwd(), "pydantic", "enum_header.py" ) - header_file = open(header_file_path, 'r') + header_file = open(header_file_path, "r") file.write(header_file.read()) with open(class_file, "a") as file: template_path = os.path.join( @@ -228,15 +253,13 @@ def run_template_enum(version_path, class_details, templates): def run_template_schema(version_path, class_details, templates): for template_info in templates: - class_file = os.path.join( - version_path, "schema" + template_info["ext"] - ) + class_file = os.path.join(version_path, "schema" + template_info["ext"]) if not os.path.exists(class_file): with open(class_file, "w") as file: schema_file_path = os.path.join( os.getcwd(), "pydantic", "schema_header.py" ) - schema_file = open(schema_file_path, 'r') + schema_file = open(schema_file_path, "r") file.write(schema_file.read()) with open(class_file, "a") as file: template_path = os.path.join( diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index d19e05b7..1e86bf7c 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -4,11 +4,12 @@ ConfigDict, Field, field_validator, -# computed_field + # computed_field ) -#from geoalchemy2.shape import to_shape -#from geoalchemy2.elements import WKBElement -#from shapely.geometry import Point + +# from geoalchemy2.shape import to_shape +# from geoalchemy2.elements import WKBElement +# from shapely.geometry import Point from datetime import date, datetime, time from typing import Optional, Iterator, List from .Base import Base @@ -87,4 +88,4 @@ def validate_point_format(cls, v): return Point(point) else: raise ValueError("must be a Point or a WKBElement") -""" \ No newline at end of file +""" From 87eacd089b59b0a1767a850d6b5c5f54eaf3a0b7 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 13:19:17 +0200 Subject: [PATCH 20/34] add support for GL profile --- pydantic/langPack.py | 3 +-- pydantic/schema_header.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 79559124..8f938c03 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -29,8 +29,7 @@ def location(version): template_files = [{"filename": "pydantic_class_template.mustache", "ext": ".py"}] enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] -required_profiles = ["EQ"] - +required_profiles = ['EQ', 'GL'] def get_class_location(class_name, class_map, version): # Check if the current class has a parent class diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index 1e86bf7c..60e0b021 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -4,21 +4,20 @@ ConfigDict, Field, field_validator, - # computed_field + computed_field ) - -# from geoalchemy2.shape import to_shape -# from geoalchemy2.elements import WKBElement -# from shapely.geometry import Point +from geoalchemy2.shape import to_shape +from geoalchemy2.elements import WKBElement +from shapely.geometry import Point from datetime import date, datetime, time from typing import Optional, Iterator, List from .Base import Base from .util import cyclic_references_validator from .enum import * -""" -class PositionPoint(Base): +class PositionPoint(Base): + """ Set of spatial coordinates that determine a point, defined in the coordinate system specified in 'Location.CoordinateSystem'. Use a single position point instance to desribe a point-oriented location. Use a sequence of position points to describe a line-oriented object (physical location of non-point oriented objects like cables or lines), or area of an object (like a substation or a geographical zone - in this case, have first and last position point with the same values). :Location: Location described by this position point. @@ -26,7 +25,7 @@ class PositionPoint(Base): :xPosition: X axis position. :yPosition: Y axis position. :zPosition: (if applicable) Z axis position. - + """ possibleProfileList: dict = Field( default={ @@ -88,4 +87,3 @@ def validate_point_format(cls, v): return Point(point) else: raise ValueError("must be a Point or a WKBElement") -""" From bf33749927ad2a747dfb576c1c282f4b6841408d Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 13:55:43 +0200 Subject: [PATCH 21/34] fix model for PositionPoint --- pydantic/langPack.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 8f938c03..1cdc80b8 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -72,7 +72,7 @@ def _set_attribute(text, render): if is_required_profile(attribute["attr_origin"]): return ( attribute["label"] - + ":" + + ": " + _set_data_type(attribute) + _set_default(attribute) ) @@ -138,12 +138,16 @@ def _compute_data_type(attribute): return attribute["range"].split("#")[1] +def _ends_with_s(attribute_name): + return attribute_name.endswith("s") + def _set_data_type(attribute): datatype = _compute_data_type(attribute) + multiplicity_by_name = _ends_with_s(attribute["label"]) if "multiplicity" in attribute: multiplicity = attribute["multiplicity"] - if multiplicity in ["M:1", "M:1..1", ""]: + if multiplicity in ["M:1", "M:1..1", ""] and not multiplicity_by_name: return datatype if multiplicity in ["M:0..1"]: return "Optional[" + datatype + "]" @@ -151,6 +155,8 @@ def _set_data_type(attribute): return "Optional[List[" + datatype + "]]" elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: return "List[" + datatype + "]" + elif multiplicity_by_name: + return "List[" + datatype + "]" else: return "List[" + datatype + "]" else: From 02d5e61c0817bdff5d615d686da953403442f68c Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 13:56:00 +0200 Subject: [PATCH 22/34] fix model for PositionPoint --- pydantic/schema_header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index 60e0b021..80c57c8c 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -50,7 +50,7 @@ class PositionPoint(Base): } ) - Location: List['Location'] + Location: 'Location' sequenceNumber: Optional[int] point: Point #= Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database From 2ab84c6a5c5a4b223538cc4b3b2498d82cd3c838 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 13:57:20 +0200 Subject: [PATCH 23/34] Update langPack.py --- pydantic/langPack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 1cdc80b8..564c8e36 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -156,6 +156,8 @@ def _set_data_type(attribute): elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: return "List[" + datatype + "]" elif multiplicity_by_name: + # Most probably there is a bug in the RDF that states multiplicity + # M:1 but should be M:1..N return "List[" + datatype + "]" else: return "List[" + datatype + "]" From c733ad632b70fd4f7ca985bda442ed30ab6b2ba1 Mon Sep 17 00:00:00 2001 From: Bot Date: Wed, 2 Aug 2023 11:56:51 +0000 Subject: [PATCH 24/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/langPack.py | 15 ++++++++------- pydantic/schema_header.py | 11 +++-------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 564c8e36..1b4f04e8 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -29,7 +29,8 @@ def location(version): template_files = [{"filename": "pydantic_class_template.mustache", "ext": ".py"}] enum_template_files = [{"filename": "pydantic_enum_template.mustache", "ext": ".py"}] -required_profiles = ['EQ', 'GL'] +required_profiles = ["EQ", "GL"] + def get_class_location(class_name, class_map, version): # Check if the current class has a parent class @@ -55,7 +56,7 @@ def _set_instances(text, render): instance = eval(render(text)) except: rendered = render(text) - rendered = rendered.replace('"','"') + rendered = rendered.replace(""", '"') instance = eval(rendered) if "label" in instance: value = instance["label"] + ' = "' + instance["label"] + '"' @@ -63,7 +64,8 @@ def _set_instances(text, render): value += " #" + instance["comment"] return value else: - return "" + return "" + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_attribute(text, render): @@ -141,6 +143,7 @@ def _compute_data_type(attribute): def _ends_with_s(attribute_name): return attribute_name.endswith("s") + def _set_data_type(attribute): datatype = _compute_data_type(attribute) multiplicity_by_name = _ends_with_s(attribute["label"]) @@ -292,10 +295,8 @@ def _create_init(path): # creates the Base class file, all classes inherit from this class def _copy_files(path): - shutil.copy(os.path.join( - os.getcwd(), "pydantic/Base.py"), path + "/Base.py") - shutil.copy(os.path.join( - os.getcwd(), "pydantic/util.py"), path + "/util.py") + shutil.copy(os.path.join(os.getcwd(), "pydantic/Base.py"), path + "/Base.py") + shutil.copy(os.path.join(os.getcwd(), "pydantic/util.py"), path + "/util.py") def resolve_headers(path): diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index 80c57c8c..dd5afa04 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -1,11 +1,6 @@ from __future__ import annotations import uuid -from pydantic import ( - ConfigDict, - Field, - field_validator, - computed_field -) +from pydantic import ConfigDict, Field, field_validator, computed_field from geoalchemy2.shape import to_shape from geoalchemy2.elements import WKBElement from shapely.geometry import Point @@ -50,9 +45,9 @@ class PositionPoint(Base): } ) - Location: 'Location' + Location: "Location" sequenceNumber: Optional[int] - point: Point #= Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + point: Point # = Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database model_config = ConfigDict(from_attributes=True) From d927b761e56c417496febad274a5209eb196162e Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 14:06:34 +0200 Subject: [PATCH 25/34] improve multiplicity handling --- pydantic/langPack.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 1b4f04e8..0d60c594 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -150,7 +150,9 @@ def _set_data_type(attribute): if "multiplicity" in attribute: multiplicity = attribute["multiplicity"] - if multiplicity in ["M:1", "M:1..1", ""] and not multiplicity_by_name: + if multiplicity in ["M:1..1"]: + return datatype + if multiplicity in ["M:1"] and not multiplicity_by_name: return datatype if multiplicity in ["M:0..1"]: return "Optional[" + datatype + "]" @@ -158,10 +160,12 @@ def _set_data_type(attribute): return "Optional[List[" + datatype + "]]" elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: return "List[" + datatype + "]" - elif multiplicity_by_name: + elif multiplicity in ["M:1"] and multiplicity_by_name: # Most probably there is a bug in the RDF that states multiplicity # M:1 but should be M:1..N return "List[" + datatype + "]" + elif multiplicity in [""]: + return datatype else: return "List[" + datatype + "]" else: From 8ca17e00f695c0bab702f9b820d419f2e37d4183 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 2 Aug 2023 14:48:51 +0200 Subject: [PATCH 26/34] fix representation configuration for point --- pydantic/schema_header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index dd5afa04..2475f58f 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -47,7 +47,7 @@ class PositionPoint(Base): Location: "Location" sequenceNumber: Optional[int] - point: Point # = Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + point: Point = Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database model_config = ConfigDict(from_attributes=True) From 4e188105443eb4a13b500d1b4f661c5c357debb1 Mon Sep 17 00:00:00 2001 From: Bot Date: Wed, 2 Aug 2023 12:49:34 +0000 Subject: [PATCH 27/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/schema_header.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index 2475f58f..23f0710f 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -47,7 +47,9 @@ class PositionPoint(Base): Location: "Location" sequenceNumber: Optional[int] - point: Point = Field(repr=False) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database + point: Point = Field( + repr=False + ) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database model_config = ConfigDict(from_attributes=True) From 74e6030f19a1118205196c9a567153abc0bb93a3 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Thu, 3 Aug 2023 09:15:14 +0200 Subject: [PATCH 28/34] use string as mrid while default value is uuid (serialized as a string) --- pydantic/langPack.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index 0d60c594..e3c86828 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -85,6 +85,8 @@ def _set_attribute(text, render): def _set_default(attribute): if "range" in attribute and "isFixed" in attribute: return " = " + attribute["range"].split("#")[1] + "." + attribute["isFixed"] + elif "label" in attribute and attribute["label"] == "mRID": + return " = Field(default_factory=uuid.uuid4)" elif "multiplicity" in attribute: multiplicity = attribute["multiplicity"] if multiplicity in ["M:1", "M:1..1"]: @@ -110,7 +112,7 @@ def _is_primitive(datatype): def _compute_data_type(attribute): if "label" in attribute and attribute["label"] == "mRID": - return "uuid.UUID" + return "str" if "dataType" in attribute: if attribute["dataType"].startswith("#"): @@ -186,6 +188,17 @@ def _set_validator(text, render): + attribute["label"] + '", mode="wrap")(cyclic_references_validator)' ) + elif attribute["label"] == "mRID": + return ( + '@field_validator("mRID", mode="before")\n' + +' def validate_mrid_format(cls, v):\n' + +' if isinstance(v, uuid.UUID):\n' + +' return str(v)\n' + +' elif isinstance(v, str):\n' + +' return v\n' + +' else:\n' + +' raise ValueError("must be a UUID or str")\n' + ) else: return "" From f19cd4aea0aacad820685853b7723e8c8aaa9b73 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Thu, 3 Aug 2023 09:30:45 +0200 Subject: [PATCH 29/34] revert change --- CIMgen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CIMgen.py b/CIMgen.py index 30821683..2387fc1c 100644 --- a/CIMgen.py +++ b/CIMgen.py @@ -529,6 +529,10 @@ def _write_files(class_details, outputPath, version): class_details["class_location"] = class_details["langPack"].base[ "class_location" ](version) + class_details["super_init"] = False + else: + # If class is a subclass a super().__init__() is needed + class_details["super_init"] = True # The entry dataType for an attribute is only set for basic data types. If the entry is not set here, the attribute # is a reference to another class and therefore the entry dataType is generated and set to the multiplicity From 3c00f69b47061a52cd14c2a24eb6d759d9bbeac5 Mon Sep 17 00:00:00 2001 From: Bot Date: Thu, 3 Aug 2023 07:15:55 +0000 Subject: [PATCH 30/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/langPack.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index e3c86828..d12c3ba2 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -191,13 +191,13 @@ def _set_validator(text, render): elif attribute["label"] == "mRID": return ( '@field_validator("mRID", mode="before")\n' - +' def validate_mrid_format(cls, v):\n' - +' if isinstance(v, uuid.UUID):\n' - +' return str(v)\n' - +' elif isinstance(v, str):\n' - +' return v\n' - +' else:\n' - +' raise ValueError("must be a UUID or str")\n' + + " def validate_mrid_format(cls, v):\n" + + " if isinstance(v, uuid.UUID):\n" + + " return str(v)\n" + + " elif isinstance(v, str):\n" + + " return v\n" + + " else:\n" + + ' raise ValueError("must be a UUID or str")\n' ) else: return "" From 4916422f09432146362408e76d5ce056c1daec6e Mon Sep 17 00:00:00 2001 From: chicco785 Date: Thu, 3 Aug 2023 07:16:33 +0000 Subject: [PATCH 31/34] docs(release_notes): update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0e5ca49a..aad7d88c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # cimgen Release Notes -## 0.0.1-dev - 2023-08-02 +## 0.0.1-dev - 2023-08-03 ### Features From 335848a1371161e8be9cd16ba2d7d376b437adb2 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Thu, 3 Aug 2023 15:58:10 +0200 Subject: [PATCH 32/34] support defaults --- pydantic/langPack.py | 30 +++++++++++++------ pydantic/schema_header.py | 4 +-- .../pydantic_class_template.mustache | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index d12c3ba2..a3aa8660 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -67,13 +67,19 @@ def _set_instances(text, render): return "" +def _lower_case_first_char(str): + return str[:1].lower() + str[1:] if str else '' + +def _set_lower_case(text, render): + return _lower_case_first_char(render(text)) + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_attribute(text, render): attribute = eval(render(text)) if is_required_profile(attribute["attr_origin"]): return ( - attribute["label"] + _lower_case_first_char(attribute["label"]) + ": " + _set_data_type(attribute) + _set_default(attribute) @@ -83,24 +89,29 @@ def _set_attribute(text, render): def _set_default(attribute): + multiplicity_by_name = _ends_with_s(attribute["label"]) if "range" in attribute and "isFixed" in attribute: return " = " + attribute["range"].split("#")[1] + "." + attribute["isFixed"] elif "label" in attribute and attribute["label"] == "mRID": return " = Field(default_factory=uuid.uuid4)" elif "multiplicity" in attribute: multiplicity = attribute["multiplicity"] + if multiplicity in ["M:1"] and multiplicity_by_name: + # Most probably there is a bug in the RDF that states multiplicity + # M:1 but should be M:1..N + return ' = Field(default=[], alias="'+attribute["label"]+'")' if multiplicity in ["M:1", "M:1..1"]: - return "" + return ' = Field(alias="'+attribute["label"]+'")' if multiplicity in ["M:0..1"]: - return "" + return ' = Field(default=None, alias="'+attribute["label"]+'")' elif multiplicity in ["M:0..n"] or "M:0.." in multiplicity: - return "" + return ' = Field(default=[], alias="'+attribute["label"]+'")' elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: - return "" + return ' = Field(default=[], alias="'+attribute["label"]+'")' else: - return "" + return ' = Field(default=[], alias="'+attribute["label"]+'")' else: - return "" + return ' = Field(alias="'+attribute["label"]+'")' def _is_primitive(datatype): @@ -183,9 +194,9 @@ def _set_validator(text, render): if not _is_primitive(datatype) and is_required_profile(attribute["attr_origin"]): return ( "val_" - + attribute["label"] + + _lower_case_first_char(attribute["label"]) + '_wrap = field_validator("' - + attribute["label"] + + _lower_case_first_char(attribute["label"]) + '", mode="wrap")(cyclic_references_validator)' ) elif attribute["label"] == "mRID": @@ -294,6 +305,7 @@ def run_template_schema(version_path, class_details, templates): ) class_details["setAttribute"] = _set_attribute class_details["setValidator"] = _set_validator + class_details["setLowerCase"] = _set_lower_case with open(template_path) as f: args = { "data": class_details, diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index 23f0710f..c6ea5b41 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -45,8 +45,8 @@ class PositionPoint(Base): } ) - Location: "Location" - sequenceNumber: Optional[int] + location: "Location" = Field(alias = "Location") + sequenceNumber: Optional[int] = Field(default=None) point: Point = Field( repr=False ) # we introduce this field compared to CIM definition because we want to store a proper geometry "point" in the database diff --git a/pydantic/templates/pydantic_class_template.mustache b/pydantic/templates/pydantic_class_template.mustache index 095e8363..f5f057e1 100644 --- a/pydantic/templates/pydantic_class_template.mustache +++ b/pydantic/templates/pydantic_class_template.mustache @@ -9,7 +9,7 @@ class {{class_name}}({{sub_class_of}}): ''' possibleProfileList: dict = Field(default={'class': [{{#class_origin}}CgmesProfileEnum.{{origin}}, {{/class_origin}}], - {{#attributes}}'{{label}}': [{{#attr_origin}}CgmesProfileEnum.{{origin}}, {{/attr_origin}}], + {{#attributes}}'{{#setLowerCase}}{{label}}{{/setLowerCase}}': [{{#attr_origin}}CgmesProfileEnum.{{origin}}, {{/attr_origin}}], {{/attributes}} }) {{#attributes}} From 230d15735c992d8101c279cbb65a4fff7ec17bc9 Mon Sep 17 00:00:00 2001 From: Bot Date: Thu, 3 Aug 2023 13:59:00 +0000 Subject: [PATCH 33/34] Automated Black fmt fixes Signed-off-by: Bot --- pydantic/langPack.py | 18 ++++++++++-------- pydantic/schema_header.py | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pydantic/langPack.py b/pydantic/langPack.py index a3aa8660..86236568 100644 --- a/pydantic/langPack.py +++ b/pydantic/langPack.py @@ -68,11 +68,13 @@ def _set_instances(text, render): def _lower_case_first_char(str): - return str[:1].lower() + str[1:] if str else '' + return str[:1].lower() + str[1:] if str else "" + def _set_lower_case(text, render): return _lower_case_first_char(render(text)) + # called by chevron, text contains the label {{dataType}}, which is evaluated by the renderer (see class template) def _set_attribute(text, render): attribute = eval(render(text)) @@ -99,19 +101,19 @@ def _set_default(attribute): if multiplicity in ["M:1"] and multiplicity_by_name: # Most probably there is a bug in the RDF that states multiplicity # M:1 but should be M:1..N - return ' = Field(default=[], alias="'+attribute["label"]+'")' + return ' = Field(default=[], alias="' + attribute["label"] + '")' if multiplicity in ["M:1", "M:1..1"]: - return ' = Field(alias="'+attribute["label"]+'")' + return ' = Field(alias="' + attribute["label"] + '")' if multiplicity in ["M:0..1"]: - return ' = Field(default=None, alias="'+attribute["label"]+'")' + return ' = Field(default=None, alias="' + attribute["label"] + '")' elif multiplicity in ["M:0..n"] or "M:0.." in multiplicity: - return ' = Field(default=[], alias="'+attribute["label"]+'")' + return ' = Field(default=[], alias="' + attribute["label"] + '")' elif multiplicity in ["M:1..n"] or "M:1.." in multiplicity: - return ' = Field(default=[], alias="'+attribute["label"]+'")' + return ' = Field(default=[], alias="' + attribute["label"] + '")' else: - return ' = Field(default=[], alias="'+attribute["label"]+'")' + return ' = Field(default=[], alias="' + attribute["label"] + '")' else: - return ' = Field(alias="'+attribute["label"]+'")' + return ' = Field(alias="' + attribute["label"] + '")' def _is_primitive(datatype): diff --git a/pydantic/schema_header.py b/pydantic/schema_header.py index c6ea5b41..36bfca7a 100644 --- a/pydantic/schema_header.py +++ b/pydantic/schema_header.py @@ -45,7 +45,7 @@ class PositionPoint(Base): } ) - location: "Location" = Field(alias = "Location") + location: "Location" = Field(alias="Location") sequenceNumber: Optional[int] = Field(default=None) point: Point = Field( repr=False From e7f77c93a44c505cb5c6009f9be5bec84b21cd13 Mon Sep 17 00:00:00 2001 From: chicco785 Date: Wed, 13 Sep 2023 18:22:56 +0000 Subject: [PATCH 34/34] docs(release_notes): update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index aad7d88c..1ce3f20f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # cimgen Release Notes -## 0.0.1-dev - 2023-08-03 +## 0.0.1-dev - 2023-09-13 ### Features