From bc0eced2a40f5595cfe97327a9d967ecf0158b90 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Fri, 23 Jul 2021 20:44:56 +0200 Subject: [PATCH 01/29] BUG: missing sdf + copy types.xsd --- sdf/1.8/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdf/1.8/CMakeLists.txt b/sdf/1.8/CMakeLists.txt index a1a70717b..5585aeb73 100644 --- a/sdf/1.8/CMakeLists.txt +++ b/sdf/1.8/CMakeLists.txt @@ -1,5 +1,6 @@ set (sdfs actor.sdf + air_pressure.sdf altimeter.sdf atmosphere.sdf audio_source.sdf @@ -23,6 +24,7 @@ set (sdfs imu.sdf inertial.sdf joint.sdf + lidar.sdf light.sdf light_state.sdf link.sdf @@ -34,6 +36,7 @@ set (sdfs model.sdf model_state.sdf noise.sdf + particle_emitter.sdf physics.sdf plane_shape.sdf plugin.sdf @@ -75,6 +78,8 @@ foreach(FIL ${sdfs}) VERBATIM) endforeach() +configure_file(schema/types.xsd ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + add_custom_target(schema1_8 ALL DEPENDS ${SDF_SCHEMA}) set_source_files_properties(${SDF_SCHEMA} PROPERTIES GENERATED TRUE) From 22bba46901bbdd411e251249154aea07c10c480e Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 25 Jul 2021 22:19:01 +0200 Subject: [PATCH 02/29] FEAT: python schema generator --- sdf/1.8/CMakeLists.txt | 2 +- sdf/1.8/schema/types.xsd | 2 +- tools/xmlschema.py | 349 +++++++++++++++++++ tools/xsd_templates/attribute.xsd | 2 + tools/xsd_templates/description.xsd | 5 + tools/xsd_templates/element.xsd | 3 + tools/xsd_templates/element_with_comment.xsd | 5 + tools/xsd_templates/expansion_type.xsd | 7 + tools/xsd_templates/file.xsd | 15 + tools/xsd_templates/import.xsd | 1 + tools/xsd_templates/include.xsd | 1 + tools/xsd_templates/type.xsd | 7 + 12 files changed, 397 insertions(+), 2 deletions(-) create mode 100644 tools/xmlschema.py create mode 100644 tools/xsd_templates/attribute.xsd create mode 100644 tools/xsd_templates/description.xsd create mode 100644 tools/xsd_templates/element.xsd create mode 100644 tools/xsd_templates/element_with_comment.xsd create mode 100644 tools/xsd_templates/expansion_type.xsd create mode 100644 tools/xsd_templates/file.xsd create mode 100644 tools/xsd_templates/import.xsd create mode 100644 tools/xsd_templates/include.xsd create mode 100644 tools/xsd_templates/type.xsd diff --git a/sdf/1.8/CMakeLists.txt b/sdf/1.8/CMakeLists.txt index 5585aeb73..530dcc83b 100644 --- a/sdf/1.8/CMakeLists.txt +++ b/sdf/1.8/CMakeLists.txt @@ -71,7 +71,7 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/tools/xmlschema.rb + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" diff --git a/sdf/1.8/schema/types.xsd b/sdf/1.8/schema/types.xsd index 53f3d8db6..e7c2e5f60 100644 --- a/sdf/1.8/schema/types.xsd +++ b/sdf/1.8/schema/types.xsd @@ -1,5 +1,5 @@ - + diff --git a/tools/xmlschema.py b/tools/xmlschema.py new file mode 100644 index 000000000..5974a6deb --- /dev/null +++ b/tools/xmlschema.py @@ -0,0 +1,349 @@ +from xml.etree import ElementTree +from pathlib import Path +import argparse +from dataclasses import dataclass +from typing import List + +def valid_path(arg:str): + path = Path(arg) + if path.exists(): + return path + else: + raise ValueError(f"The path does not exist: {arg}") + + +def existing_dir(arg:str): + path = valid_path(arg) + if path.is_dir(): + return path + else: + raise ValueError(f"Not a directory: {arg}") + +def existing_file(arg:str): + path = valid_path(arg) + if path.is_file(): + return path + else: + raise ValueError(f"Not a file: {arg}") + +def template_path(template_path:str): + if template_path == "": + return Path(__file__).parent / "xsd_templates" + else: + return existing_dir(template_path) + +cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") +cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=existing_file, help="SDF file to compile.") +cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=existing_dir, help="Directory containing all the SDF files.") +cmd_arg_parser.add_argument("-o", "--out", dest="target", default=".", type=str, help="Output directory for xsd file. Will be created if it doesn't exit.") +cmd_arg_parser.add_argument("--template-dir", dest="templates", default="", type=template_path, help="Location to search for xsd tempate files. Default: /tools/xsd_templates.") +cmd_arg_parser.add_argument("--ns-prefix", dest="ns_prefix", default="sdformat", type=str, help="Prefix for generated xsd namespaces.") + +args = cmd_arg_parser.parse_args() +source:Path = args.source +source_dir:Path = args.directory +xsd_file_template:str = (args.templates / "file.xsd").read_text() + + + +def _tabulate(input:str, offset:int=0) -> str: + formatted = "" + for line in input.split("\n")[:-1]: + formatted += (" " * 4) * offset + line + "\n" + return formatted + +# collect existing namespaces +namespaces = {"types": f"{args.ns_prefix}/types"} +for path in source_dir.iterdir(): + if not path.is_file(): + continue + + if not path.suffix == ".sdf": + continue + + namespaces[path.stem] = f"{args.ns_prefix}/{path.stem}" + +@dataclass +class Element: + name: str + type: str + required:str + default:str=None + description:str=None + + def to_xsd(self): + return f"" + + def to_typedef(self): + """The string used inside a ComplexType to refer to this element""" + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.required] + + if self.description: + template = (args.templates / "element_with_comment.xsd").read_text() + else: + template = (args.templates / "element.xsd").read_text() + template = template.format( + min_occurs=min_occurs, + max_occurs=max_occurs, + name = self.name, + type = self.type, + default = f"default='{self.default}'" if self.default is not None else "", + description = self.description, + ) + + return _tabulate(template, 2) + +@dataclass +class Attribute: + name: str + type: str + required: str + default: int + description:str=None + + def to_typedef(self): + template = (args.templates / "attribute.xsd").read_text() + template = template.format( + name = self.name, + type = self.type, + required = "required" if self.required == "1" else "optional", + default = self.default + ) + + return _tabulate(template, 1) + +@dataclass +class ComplexType: + name: str + element: ElementTree.Element + + def to_xsd(self, declared): + attributes = list() + elements = list() + + for attribute in self.element.findall("attribute"): + if "default" in attribute.attrib.keys(): + default = attribute.attrib["default"] + + attributes.append(Attribute( + name = attribute.attrib["name"], + type = attribute.attrib["type"], + required= attribute.attrib["required"], + default=default + ).to_typedef()) + + for child in self.element.findall("element"): + if "copy_data" in child.attrib: + # special element for plugin.sdf to allow + # declare that any children are allowed + any_xsd = "\n" + elements.append(_tabulate(any_xsd, 2)) + continue + + name = child.attrib["name"] + elements.append(declared["elements"][name].to_typedef()) + + for child in self.element.findall("include"): + child:ElementTree.Element + other_file = source_dir / child.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + elements.append(declared["elements"][name].to_typedef()) + + if "type" in self.element.attrib: + if len(elements) > 0: + raise RuntimeError("The compiler cant generate this type.") + + template = (args.templates / "expansion_type.xsd").read_text() + template = template.format( + name = self.name, + type = self.element.attrib["type"], + attributes = "\n".join(attributes) + ) + + else: + elements = "\n".join(elements) + attributes = "\n".join(attributes) + + template = (args.templates / "type.xsd").read_text() + template = template.format( + name = self.name, + elements = elements, + attributes = attributes + ) + + return template + +@dataclass +class SimpleType: + name: str + namespace:str=None + + def __eq__(self, other): + return self.name == other.name + + def __hash__(self): + return hash(self.name) + + def to_xsd(self): + name = self.name + known_types = { + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string":"xs:string", + "double":"xs:double", + "int":"xs:int", + "float":"xs:float", + "char":"xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } + + + try: + referred = known_types[name] + except KeyError: + raise RuntimeError(f"Unknown primitive type: {name}") from None + + return f"" + +# parse file +# recurse elements of the file and track which xsd elements need to be generated +def expand(element:ElementTree.Element, declared:dict): + if element.tag == "description": + # not handled at this stage and may + # contain tags (see particle_emitter) + return + + for child in element: + expand(child, declared) + + desc_element = element.find("description") + description = desc_element.text if desc_element else None + + if element.tag == "element": + if "copy_data" in element.attrib: + # special element for plugin.sdf + # essentially allows any element to occur here + # we ignore it here, but insert while creating + # the pluginType + return + + name = element.attrib["name"] + element_type = name + "Type" + required = element.attrib["required"] + + num_children = len(element) + if element.find("description") is not None: + num_children = len(element) - 1 + + has_children = num_children > 0 + has_type = "type" in element.attrib + + if has_children and has_type: + # extens existing simple type + # Example: pose in pose.sdf + element_type = name + "Type" + declared.setdefault("simple_types", set()).add(SimpleType(element.attrib["type"])) + declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) + elif has_type: + # redefinition of simple type + # Example: (some) children of link.sdf + element_type = element.attrib["type"] + declared.setdefault("simple_types", set()).add(SimpleType(element_type)) + elif has_children: + # new complex type + # Example: world in world.sdf + element_type = name + "Type" + declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) + else: + # element that wraps a string + # Example: audio_sink in audio_sink.sdf + element_type = name + "Type" + # declared.setdefault("simple_types", set()).add(SimpleType("string")) + declared.setdefault("complex_types", dict())[element_type] = ComplexType(name + "Type", element) + + + default = element.attrib["default"] if "default" in element.attrib else None + elements:dict = declared.setdefault("elements", dict()) + elements[name] = Element( + name, + type=element_type, + required=required, + default=default, + description=description + ) + elif element.tag == "include": + other_file = source_dir / element.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + element_type = f"{other_file.stem}:{name}Type" + required = element.attrib["required"] + elements = declared.setdefault("elements", dict()) + elements[name] = Element(name, element_type, required) + declared.setdefault("imports", set()).add(other_file.stem) + elif element.tag == "attribute": + element_type = element.attrib["type"] + declared.setdefault("simple_types", set()).add(SimpleType(element_type)) + else: + raise RuntimeError(f"Unknown SDF element encountered: {element.tag}") + +root = ElementTree.parse(source).getroot() +declared = dict() +declared.setdefault("imports", set()).add(source.stem) +expand(root, declared) + + +strings = dict() + +for name in declared["imports"]: + ns_elements = strings.setdefault("namespaces", []) + ns_elements.append(f" xmlns:{name}='{namespaces[name]}'") + imp_elements = strings.setdefault("imports", []) + imp_elements.append(f" ") + +if "simple_types" in declared: + for simple_type in declared["simple_types"]: + elements = strings.setdefault("simple_types", []) + elements.append(simple_type.to_xsd()) +else: + strings["simple_types"] = list() + +if "complex_types" in declared: + for complex_type in declared["complex_types"].values(): + elements = strings.setdefault("complex_types", []) + elements.append(complex_type.to_xsd(declared)) +else: + strings["complex_types"] = list() + + + +# write the file to disk +substitutions = { + "file_namespace": namespaces[source.stem], + "namespaces": "\n".join(strings["namespaces"]), + "imports":"\n".join(strings["imports"]), + "element":declared["elements"][root.attrib["name"]].to_xsd(), + "simple_types":"\n".join(strings["simple_types"]), + "complex_types":"\n".join(strings["complex_types"]) +} + +out_dir = Path(args.target) +if not out_dir.exists(): + out_dir.mkdir(exist_ok=True, parents=True) + +with open(out_dir / (source.stem + ".xsd"), "w") as out_file: + print(xsd_file_template.format(**substitutions), file=out_file) diff --git a/tools/xsd_templates/attribute.xsd b/tools/xsd_templates/attribute.xsd new file mode 100644 index 000000000..017d812f7 --- /dev/null +++ b/tools/xsd_templates/attribute.xsd @@ -0,0 +1,2 @@ + + diff --git a/tools/xsd_templates/description.xsd b/tools/xsd_templates/description.xsd new file mode 100644 index 000000000..23e703c7b --- /dev/null +++ b/tools/xsd_templates/description.xsd @@ -0,0 +1,5 @@ + + + <![CDATA[{INNER_TEXT}]]> + + diff --git a/tools/xsd_templates/element.xsd b/tools/xsd_templates/element.xsd new file mode 100644 index 000000000..701d170a6 --- /dev/null +++ b/tools/xsd_templates/element.xsd @@ -0,0 +1,3 @@ + + + diff --git a/tools/xsd_templates/element_with_comment.xsd b/tools/xsd_templates/element_with_comment.xsd new file mode 100644 index 000000000..a1526be89 --- /dev/null +++ b/tools/xsd_templates/element_with_comment.xsd @@ -0,0 +1,5 @@ + + + {description} + + diff --git a/tools/xsd_templates/expansion_type.xsd b/tools/xsd_templates/expansion_type.xsd new file mode 100644 index 000000000..31d60889a --- /dev/null +++ b/tools/xsd_templates/expansion_type.xsd @@ -0,0 +1,7 @@ + + + +{attributes} + + + diff --git a/tools/xsd_templates/file.xsd b/tools/xsd_templates/file.xsd new file mode 100644 index 000000000..f3660599c --- /dev/null +++ b/tools/xsd_templates/file.xsd @@ -0,0 +1,15 @@ + + + +{imports} + +{element} + +{simple_types} + +{complex_types} + + diff --git a/tools/xsd_templates/import.xsd b/tools/xsd_templates/import.xsd new file mode 100644 index 000000000..6bd2840b3 --- /dev/null +++ b/tools/xsd_templates/import.xsd @@ -0,0 +1 @@ + diff --git a/tools/xsd_templates/include.xsd b/tools/xsd_templates/include.xsd new file mode 100644 index 000000000..ed7359a6d --- /dev/null +++ b/tools/xsd_templates/include.xsd @@ -0,0 +1 @@ + diff --git a/tools/xsd_templates/type.xsd b/tools/xsd_templates/type.xsd new file mode 100644 index 000000000..691220cfa --- /dev/null +++ b/tools/xsd_templates/type.xsd @@ -0,0 +1,7 @@ + + +{elements} + + +{attributes} + From 2d4a4573ebb0937a97b0b7c44f77052039c4ec54 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 25 Jul 2021 22:29:47 +0200 Subject: [PATCH 03/29] MAINT: remove (unnecessary) validators --- sdf/1.8/CMakeLists.txt | 2 +- tools/xmlschema.py | 51 +++++++++--------------------------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/sdf/1.8/CMakeLists.txt b/sdf/1.8/CMakeLists.txt index 530dcc83b..e977a825f 100644 --- a/sdf/1.8/CMakeLists.txt +++ b/sdf/1.8/CMakeLists.txt @@ -72,7 +72,7 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py - ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} + ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" VERBATIM) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 5974a6deb..cb4307092 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -4,46 +4,17 @@ from dataclasses import dataclass from typing import List -def valid_path(arg:str): - path = Path(arg) - if path.exists(): - return path - else: - raise ValueError(f"The path does not exist: {arg}") - - -def existing_dir(arg:str): - path = valid_path(arg) - if path.is_dir(): - return path - else: - raise ValueError(f"Not a directory: {arg}") - -def existing_file(arg:str): - path = valid_path(arg) - if path.is_file(): - return path - else: - raise ValueError(f"Not a file: {arg}") - -def template_path(template_path:str): - if template_path == "": - return Path(__file__).parent / "xsd_templates" - else: - return existing_dir(template_path) - cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") -cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=existing_file, help="SDF file to compile.") -cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=existing_dir, help="Directory containing all the SDF files.") +cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") +cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") cmd_arg_parser.add_argument("-o", "--out", dest="target", default=".", type=str, help="Output directory for xsd file. Will be created if it doesn't exit.") -cmd_arg_parser.add_argument("--template-dir", dest="templates", default="", type=template_path, help="Location to search for xsd tempate files. Default: /tools/xsd_templates.") cmd_arg_parser.add_argument("--ns-prefix", dest="ns_prefix", default="sdformat", type=str, help="Prefix for generated xsd namespaces.") args = cmd_arg_parser.parse_args() -source:Path = args.source -source_dir:Path = args.directory -xsd_file_template:str = (args.templates / "file.xsd").read_text() - +source_dir = Path(args.directory) +source:Path = source_dir / args.source +template_dir = Path(__file__).parent / "xsd_templates" +xsd_file_template:str = (template_dir / "file.xsd").read_text() def _tabulate(input:str, offset:int=0) -> str: @@ -87,9 +58,9 @@ def to_typedef(self): min_occurs, max_occurs = required_codes[self.required] if self.description: - template = (args.templates / "element_with_comment.xsd").read_text() + template = (template_dir / "element_with_comment.xsd").read_text() else: - template = (args.templates / "element.xsd").read_text() + template = (template_dir / "element.xsd").read_text() template = template.format( min_occurs=min_occurs, max_occurs=max_occurs, @@ -110,7 +81,7 @@ class Attribute: description:str=None def to_typedef(self): - template = (args.templates / "attribute.xsd").read_text() + template = (template_dir / "attribute.xsd").read_text() template = template.format( name = self.name, type = self.type, @@ -162,7 +133,7 @@ def to_xsd(self, declared): if len(elements) > 0: raise RuntimeError("The compiler cant generate this type.") - template = (args.templates / "expansion_type.xsd").read_text() + template = (template_dir / "expansion_type.xsd").read_text() template = template.format( name = self.name, type = self.element.attrib["type"], @@ -173,7 +144,7 @@ def to_xsd(self, declared): elements = "\n".join(elements) attributes = "\n".join(attributes) - template = (args.templates / "type.xsd").read_text() + template = (template_dir / "type.xsd").read_text() template = template.format( name = self.name, elements = elements, From 4548b82423520c98282a7f7a8a76c62dbf8ed95c Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Tue, 27 Jul 2021 20:13:44 +0200 Subject: [PATCH 04/29] MAINT: use etree for xml generation --- tools/xmlschema.py | 220 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 185 insertions(+), 35 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index cb4307092..c390d7899 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -1,9 +1,19 @@ from xml.etree import ElementTree + from pathlib import Path import argparse from dataclasses import dataclass from typing import List + +def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: + """Given XML/XSD input, add indentation and return it""" + formatted = "" + for line in input.split("\n")[:-1]: + formatted += style * offset + line + "\n" + return formatted + + cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") @@ -16,15 +26,11 @@ template_dir = Path(__file__).parent / "xsd_templates" xsd_file_template:str = (template_dir / "file.xsd").read_text() - -def _tabulate(input:str, offset:int=0) -> str: - formatted = "" - for line in input.split("\n")[:-1]: - formatted += (" " * 4) * offset + line + "\n" - return formatted - # collect existing namespaces -namespaces = {"types": f"{args.ns_prefix}/types"} +namespaces = { + "types": "http://sdformat.org/schemas/types.xsd", + "xs": "http://www.w3.org/2001/XMLSchema" +} for path in source_dir.iterdir(): if not path.is_file(): continue @@ -33,6 +39,17 @@ def _tabulate(input:str, offset:int=0) -> str: continue namespaces[path.stem] = f"{args.ns_prefix}/{path.stem}" + ElementTree.register_namespace(path.stem, f"{args.ns_prefix}/{path.stem}") + + +def _to_qname(name:str) -> str: + try: + prefix, name = name.split(":") + except ValueError: + # no prefix + return name + else: + return f"{{{namespaces[prefix]}}}{name}" @dataclass class Element: @@ -42,9 +59,6 @@ class Element: default:str=None description:str=None - def to_xsd(self): - return f"" - def to_typedef(self): """The string used inside a ComplexType to refer to this element""" @@ -70,7 +84,39 @@ def to_typedef(self): description = self.description, ) - return _tabulate(template, 2) + return template + + def to_basic(self): + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", self.name) + el.set("type", self.type) + if self.default: + el.set("default", self.default) + return el + + def to_etree_element(self): + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", self.name) + el.set("type", self.type) + if self.default: + el.set("default", self.default) + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.required] + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.tail = "\n" + choice.set("minOccurs", min_occurs) + choice.set("maxOccurs", max_occurs) + choice.append(el) + + return choice + @dataclass class Attribute: @@ -89,7 +135,20 @@ def to_typedef(self): default = self.default ) - return _tabulate(template, 1) + return template + + def to_etree_element(self): + el = ElementTree.Element(_to_qname("xs:attribute")) + el.tail = "\n" + + el.set("name", self.name) + el.set("type", self.type) + el.set("use", "required" if self.required == "1" else "optional") + + if self.default is not None: + el.set("default", self.default) + + return el @dataclass class ComplexType: @@ -104,12 +163,12 @@ def to_xsd(self, declared): if "default" in attribute.attrib.keys(): default = attribute.attrib["default"] - attributes.append(Attribute( + attributes.append(_tabulate(Attribute( name = attribute.attrib["name"], type = attribute.attrib["type"], required= attribute.attrib["required"], default=default - ).to_typedef()) + ).to_typedef(), 1)) for child in self.element.findall("element"): if "copy_data" in child.attrib: @@ -120,14 +179,14 @@ def to_xsd(self, declared): continue name = child.attrib["name"] - elements.append(declared["elements"][name].to_typedef()) + elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) for child in self.element.findall("include"): child:ElementTree.Element other_file = source_dir / child.attrib["filename"] other_root = ElementTree.parse(other_file).getroot() name = other_root.attrib["name"] - elements.append(declared["elements"][name].to_typedef()) + elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) if "type" in self.element.attrib: if len(elements) > 0: @@ -153,6 +212,58 @@ def to_xsd(self, declared): return template + def to_etree_element(self, declared): + elements = list() + attributes = list() + + for attribute in self.element.findall("attribute"): + if "default" in attribute.attrib.keys(): + default = attribute.attrib["default"] + + attributes.append(Attribute( + name = attribute.attrib["name"], + type = attribute.attrib["type"], + required= attribute.attrib["required"], + default=default + ).to_etree_element()) + + for child in self.element.findall("element"): + if "copy_data" in child.attrib: + # special element for plugin.sdf to allow + # declare that any children are allowed + anyEl = ElementTree.Element(_to_qname("xs:any")) + anyEl.set("minOccurs", "0") + anyEl.set("maxOccurs", "unbounded") + anyEl.set("processContents", "skip") + elements.append(anyEl) + continue + + name = child.attrib["name"] + elements.append(declared["elements"][name].to_etree_element()) + + for child in self.element.findall("include"): + child:ElementTree.Element + other_file = source_dir / child.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + elements.append(declared["elements"][name].to_etree_element()) + + el = ElementTree.Element(_to_qname("xs:complexType")) + el.set("name", self.name) + el.tail = "\n" + + if elements: + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.set("maxOccurs", "unbounded") + choice.tail = "\n" + choice.extend(elements) + el.append(choice) + + el.extend(attributes) + + return el + + @dataclass class SimpleType: name: str @@ -191,6 +302,40 @@ def to_xsd(self): return f"" + def to_etree_element(self): + name = self.name + known_types = { + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string":"xs:string", + "double":"xs:double", + "int":"xs:int", + "float":"xs:float", + "char":"xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } + + try: + referred = known_types[name] + except KeyError: + raise RuntimeError(f"Unknown primitive type: {name}") from None + + restriction = ElementTree.Element(_to_qname("xs:restriction")) + restriction.set("base", referred) + + el = ElementTree.Element(_to_qname("xs:simpleType")) + el.set("name", name) + el.append(restriction) + el.tail = "\n" + + return el + # parse file # recurse elements of the file and track which xsd elements need to be generated def expand(element:ElementTree.Element, declared:dict): @@ -278,43 +423,48 @@ def expand(element:ElementTree.Element, declared:dict): expand(root, declared) +xsd_root = ElementTree.Element(_to_qname("xs:schema")) +xsd_root.set("xmlns", namespaces[source.stem]) +xsd_root.set("targetNamespace", namespaces[source.stem]) +xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) strings = dict() for name in declared["imports"]: - ns_elements = strings.setdefault("namespaces", []) - ns_elements.append(f" xmlns:{name}='{namespaces[name]}'") - imp_elements = strings.setdefault("imports", []) - imp_elements.append(f" ") + el = ElementTree.Element(_to_qname("xs:import")) + el.set("namespace", namespaces[name]) + el.set("schemaLocation", f"./{name}.xsd") + el.tail = "\n" + + xsd_root.append(el) + xsd_root.set(f"xmlns:{name}", namespaces[name]) if "simple_types" in declared: + el = ElementTree.Element(_to_qname("xs:import")) + el.set("namespace", namespaces["types"]) + el.set("schemaLocation", f"./types.xsd") + el.tail = "\n" + + xsd_root.append(el) + xsd_root.set(f"xmlns:types", namespaces["types"]) + for simple_type in declared["simple_types"]: - elements = strings.setdefault("simple_types", []) - elements.append(simple_type.to_xsd()) + xsd_root.append(simple_type.to_etree_element()) else: strings["simple_types"] = list() if "complex_types" in declared: for complex_type in declared["complex_types"].values(): - elements = strings.setdefault("complex_types", []) - elements.append(complex_type.to_xsd(declared)) + xsd_root.append(complex_type.to_etree_element(declared)) else: strings["complex_types"] = list() # write the file to disk -substitutions = { - "file_namespace": namespaces[source.stem], - "namespaces": "\n".join(strings["namespaces"]), - "imports":"\n".join(strings["imports"]), - "element":declared["elements"][root.attrib["name"]].to_xsd(), - "simple_types":"\n".join(strings["simple_types"]), - "complex_types":"\n".join(strings["complex_types"]) -} - out_dir = Path(args.target) if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: - print(xsd_file_template.format(**substitutions), file=out_file) + ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") + # print(xsd_string, file=out_file) From 55396a8c4daacf672e7f445cb46b1c7e2559add7 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Tue, 27 Jul 2021 21:11:01 +0200 Subject: [PATCH 05/29] MAINT: remove refactored files --- tools/xmlschema.py | 156 ++++--------------- tools/xsd_templates/attribute.xsd | 2 - tools/xsd_templates/description.xsd | 5 - tools/xsd_templates/element.xsd | 3 - tools/xsd_templates/element_with_comment.xsd | 5 - tools/xsd_templates/expansion_type.xsd | 7 - tools/xsd_templates/file.xsd | 15 -- tools/xsd_templates/import.xsd | 1 - tools/xsd_templates/include.xsd | 1 - tools/xsd_templates/type.xsd | 7 - 10 files changed, 26 insertions(+), 176 deletions(-) delete mode 100644 tools/xsd_templates/attribute.xsd delete mode 100644 tools/xsd_templates/description.xsd delete mode 100644 tools/xsd_templates/element.xsd delete mode 100644 tools/xsd_templates/element_with_comment.xsd delete mode 100644 tools/xsd_templates/expansion_type.xsd delete mode 100644 tools/xsd_templates/file.xsd delete mode 100644 tools/xsd_templates/import.xsd delete mode 100644 tools/xsd_templates/include.xsd delete mode 100644 tools/xsd_templates/type.xsd diff --git a/tools/xmlschema.py b/tools/xmlschema.py index c390d7899..f97d09511 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -24,7 +24,6 @@ def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: source_dir = Path(args.directory) source:Path = source_dir / args.source template_dir = Path(__file__).parent / "xsd_templates" -xsd_file_template:str = (template_dir / "file.xsd").read_text() # collect existing namespaces namespaces = { @@ -59,33 +58,6 @@ class Element: default:str=None description:str=None - def to_typedef(self): - """The string used inside a ComplexType to refer to this element""" - - required_codes = { - "0" : ("0", "1"), - "1" : ("1", "1"), - "+" : ("1", "unbounded"), - "*" : ("0", "unbounded"), - "-1" : ("0", "0") - } - min_occurs, max_occurs = required_codes[self.required] - - if self.description: - template = (template_dir / "element_with_comment.xsd").read_text() - else: - template = (template_dir / "element.xsd").read_text() - template = template.format( - min_occurs=min_occurs, - max_occurs=max_occurs, - name = self.name, - type = self.type, - default = f"default='{self.default}'" if self.default is not None else "", - description = self.description, - ) - - return template - def to_basic(self): el = ElementTree.Element(_to_qname("xs:element")) el.set("name", self.name) @@ -126,17 +98,6 @@ class Attribute: default: int description:str=None - def to_typedef(self): - template = (template_dir / "attribute.xsd").read_text() - template = template.format( - name = self.name, - type = self.type, - required = "required" if self.required == "1" else "optional", - default = self.default - ) - - return template - def to_etree_element(self): el = ElementTree.Element(_to_qname("xs:attribute")) el.tail = "\n" @@ -155,63 +116,6 @@ class ComplexType: name: str element: ElementTree.Element - def to_xsd(self, declared): - attributes = list() - elements = list() - - for attribute in self.element.findall("attribute"): - if "default" in attribute.attrib.keys(): - default = attribute.attrib["default"] - - attributes.append(_tabulate(Attribute( - name = attribute.attrib["name"], - type = attribute.attrib["type"], - required= attribute.attrib["required"], - default=default - ).to_typedef(), 1)) - - for child in self.element.findall("element"): - if "copy_data" in child.attrib: - # special element for plugin.sdf to allow - # declare that any children are allowed - any_xsd = "\n" - elements.append(_tabulate(any_xsd, 2)) - continue - - name = child.attrib["name"] - elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) - - for child in self.element.findall("include"): - child:ElementTree.Element - other_file = source_dir / child.attrib["filename"] - other_root = ElementTree.parse(other_file).getroot() - name = other_root.attrib["name"] - elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) - - if "type" in self.element.attrib: - if len(elements) > 0: - raise RuntimeError("The compiler cant generate this type.") - - template = (template_dir / "expansion_type.xsd").read_text() - template = template.format( - name = self.name, - type = self.element.attrib["type"], - attributes = "\n".join(attributes) - ) - - else: - elements = "\n".join(elements) - attributes = "\n".join(attributes) - - template = (template_dir / "type.xsd").read_text() - template = template.format( - name = self.name, - elements = elements, - attributes = attributes - ) - - return template - def to_etree_element(self, declared): elements = list() attributes = list() @@ -252,7 +156,15 @@ def to_etree_element(self, declared): el.set("name", self.name) el.tail = "\n" - if elements: + if elements and "type" in self.element.attrib: + pass + elif "type" in self.element.attrib: + extension = ElementTree.Element(_to_qname("xs:extension")) + extension.set("base", self.element.attrib["type"]) + simple_content = ElementTree.Element(_to_qname("xs:simpleContent")) + simple_content.append(extension) + el.append(simple_content) + elif elements: choice = ElementTree.Element(_to_qname("xs:choice")) choice.set("maxOccurs", "unbounded") choice.tail = "\n" @@ -275,33 +187,6 @@ def __eq__(self, other): def __hash__(self): return hash(self.name) - def to_xsd(self): - name = self.name - known_types = { - "unsigned int": "xs:unsignedInt", - "unsigned long": "xs:unsignedLong", - "bool": "xs:boolean", - "string":"xs:string", - "double":"xs:double", - "int":"xs:int", - "float":"xs:float", - "char":"xs:char", - "vector3": "types:vector3", - "vector2d": "types:vector2d", - "vector2i": "types:vector2i", - "pose": "types:pose", - "time": "types:time", - "color": "types:color", - } - - - try: - referred = known_types[name] - except KeyError: - raise RuntimeError(f"Unknown primitive type: {name}") from None - - return f"" - def to_etree_element(self): name = self.name known_types = { @@ -426,13 +311,12 @@ def expand(element:ElementTree.Element, declared:dict): xsd_root = ElementTree.Element(_to_qname("xs:schema")) xsd_root.set("xmlns", namespaces[source.stem]) xsd_root.set("targetNamespace", namespaces[source.stem]) -xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) strings = dict() for name in declared["imports"]: el = ElementTree.Element(_to_qname("xs:import")) el.set("namespace", namespaces[name]) - el.set("schemaLocation", f"./{name}.xsd") + el.set("schemaLocation", f"./{name}Type.xsd") el.tail = "\n" xsd_root.append(el) @@ -458,13 +342,25 @@ def expand(element:ElementTree.Element, declared:dict): else: strings["complex_types"] = list() - - -# write the file to disk out_dir = Path(args.target) + if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) +# write type file +with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: + ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") +# write element file +xsd_root = ElementTree.Element(_to_qname("xs:schema")) +# xsd_root.set("targetNamespace", namespaces[source.stem]) +xsd_root.set("xmlns", namespaces[source.stem]) +name = source.stem +el = ElementTree.Element(_to_qname("xs:import")) +el.set("namespace", namespaces[name]) +el.set("schemaLocation", f"./{name}.xsd") +el.tail = "\n" +xsd_root.append(el) +# xsd_root.set(f"xmlns:{name}", namespaces[name]) +xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") - # print(xsd_string, file=out_file) diff --git a/tools/xsd_templates/attribute.xsd b/tools/xsd_templates/attribute.xsd deleted file mode 100644 index 017d812f7..000000000 --- a/tools/xsd_templates/attribute.xsd +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tools/xsd_templates/description.xsd b/tools/xsd_templates/description.xsd deleted file mode 100644 index 23e703c7b..000000000 --- a/tools/xsd_templates/description.xsd +++ /dev/null @@ -1,5 +0,0 @@ - - - <![CDATA[{INNER_TEXT}]]> - - diff --git a/tools/xsd_templates/element.xsd b/tools/xsd_templates/element.xsd deleted file mode 100644 index 701d170a6..000000000 --- a/tools/xsd_templates/element.xsd +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/tools/xsd_templates/element_with_comment.xsd b/tools/xsd_templates/element_with_comment.xsd deleted file mode 100644 index a1526be89..000000000 --- a/tools/xsd_templates/element_with_comment.xsd +++ /dev/null @@ -1,5 +0,0 @@ - - - {description} - - diff --git a/tools/xsd_templates/expansion_type.xsd b/tools/xsd_templates/expansion_type.xsd deleted file mode 100644 index 31d60889a..000000000 --- a/tools/xsd_templates/expansion_type.xsd +++ /dev/null @@ -1,7 +0,0 @@ - - - -{attributes} - - - diff --git a/tools/xsd_templates/file.xsd b/tools/xsd_templates/file.xsd deleted file mode 100644 index f3660599c..000000000 --- a/tools/xsd_templates/file.xsd +++ /dev/null @@ -1,15 +0,0 @@ - - - -{imports} - -{element} - -{simple_types} - -{complex_types} - - diff --git a/tools/xsd_templates/import.xsd b/tools/xsd_templates/import.xsd deleted file mode 100644 index 6bd2840b3..000000000 --- a/tools/xsd_templates/import.xsd +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/xsd_templates/include.xsd b/tools/xsd_templates/include.xsd deleted file mode 100644 index ed7359a6d..000000000 --- a/tools/xsd_templates/include.xsd +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/xsd_templates/type.xsd b/tools/xsd_templates/type.xsd deleted file mode 100644 index 691220cfa..000000000 --- a/tools/xsd_templates/type.xsd +++ /dev/null @@ -1,7 +0,0 @@ - - -{elements} - - -{attributes} - From 302d66ebab8b502deffbdfc7e0ad06453bd3c874 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 09:58:37 +0200 Subject: [PATCH 06/29] MAINT: integrate recursion into class structure --- tools/xmlschema.py | 498 ++++++++++++++++++++++++--------------------- 1 file changed, 261 insertions(+), 237 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index f97d09511..0906e00fd 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -6,14 +6,6 @@ from typing import List -def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: - """Given XML/XSD input, add indentation and return it""" - formatted = "" - for line in input.split("\n")[:-1]: - formatted += style * offset + line + "\n" - return formatted - - cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") @@ -25,6 +17,34 @@ def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: source:Path = source_dir / args.source template_dir = Path(__file__).parent / "xsd_templates" +root = ElementTree.parse(source).getroot() +declared = dict() +declared["imports"] = set() + + +def _to_simple_type(in_type:str) -> str: + known_types = { + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string":"xs:string", + "double":"xs:double", + "int":"xs:int", + "float":"xs:float", + "char":"xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } + + try: + return known_types[in_type] + except KeyError: + raise RuntimeError(f"Unknown simple type: {in_type}") from None + # collect existing namespaces namespaces = { "types": "http://sdformat.org/schemas/types.xsd", @@ -50,29 +70,61 @@ def _to_qname(name:str) -> str: else: return f"{{{namespaces[prefix]}}}{name}" + +@dataclass +class Description: + element: ElementTree.Element + + def to_subtree(self): + desc_text = self.element.text + if desc_text is None or desc_text == "": + return list(), None + else: + desc_text = self.element.text.strip() + + documentation = ElementTree.Element(_to_qname("xs:documentation")) + documentation.text = desc_text + documentation.set("xml:lang", "en") + annotation = ElementTree.Element(_to_qname("xs:annotation")) + annotation.append(documentation) + return list(), annotation + @dataclass class Element: - name: str - type: str - required:str - default:str=None - description:str=None + element: ElementTree.Element def to_basic(self): + namespaces = list() el = ElementTree.Element(_to_qname("xs:element")) - el.set("name", self.name) - el.set("type", self.type) - if self.default: - el.set("default", self.default) - return el + el.set("name", self.element.attrib["name"]) + if "type" in self.element.attrib: + el.set("type", self.element.attrib["type"]) + if "default" in self.element.attrib: + el.set("default", self.element.attrib["default"]) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + return namespaces, el def to_etree_element(self): + namespaces = list() el = ElementTree.Element(_to_qname("xs:element")) el.set("name", self.name) el.set("type", self.type) if self.default: el.set("default", self.default) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.append(doc_namespaces) + if doc_el: + el.append(doc_el) + + required_codes = { "0" : ("0", "1"), "1" : ("1", "1"), @@ -80,267 +132,239 @@ def to_etree_element(self): "*" : ("0", "unbounded"), "-1" : ("0", "0") } - min_occurs, max_occurs = required_codes[self.required] + min_occurs, max_occurs = required_codes[self.element.attrib["required"]] choice = ElementTree.Element(_to_qname("xs:choice")) - choice.tail = "\n" choice.set("minOccurs", min_occurs) choice.set("maxOccurs", max_occurs) choice.append(el) return choice + def to_subtree(self): + namespaces = list() + + if "copy_data" in self.element.attrib: + # special element for plugin.sdf to + # declare that any children are allowed + anyEl = ElementTree.Element(_to_qname("xs:any")) + anyEl.set("minOccurs", "0") + anyEl.set("maxOccurs", "unbounded") + anyEl.set("processContents", "skip") + + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + anyEl.append(doc_el) + + return namespaces, anyEl + + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", self.element.attrib["name"]) + + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + + num_children = len(self.element) - len(self.element.findall("description")) + has_children = num_children > 0 + has_type = "type" in self.element.attrib + + if has_type and has_children: + child_ns, child_el = ComplexType(self.element).to_subtree() + namespaces.extend(child_ns) + el.append(child_el) + elif has_children: + child_ns, child_el = ComplexType(self.element).to_subtree() + namespaces.extend(child_ns) + el.append(child_el) + elif has_type: + child_type = _to_simple_type(self.element.attrib["type"]) + el.set("type", child_type) + if child_type.startswith("types"): + namespaces.append("types") + else: + el.set("type", _to_simple_type("string")) + + if "default" in self.element.attrib: + el.set("default", self.element.attrib["default"]) + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.element.attrib["required"]] + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.set("minOccurs", min_occurs) + choice.set("maxOccurs", max_occurs) + choice.append(el) + + return namespaces, el + + +@dataclass +class Include: + element: ElementTree.Element + + def to_subtree(self): + namespaces = list() + other_file = source_dir / self.element.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", other_root.attrib["name"]) + + el.set("type", f"{other_file.stem}:{name}Type") + namespaces.append(f"{other_file.stem}") + + if "default" in other_root.attrib: + el.set("default", other_root.attrib["default"]) + + docs = other_root.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + + if self.element.attrib["required"] == "1": + el.set("type", f"{other_file.stem}:{name}Type") + + #TODO: remove this line + declared.setdefault("imports", set()).add(other_file.stem) + + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.element.attrib["required"]] + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.set("minOccurs", min_occurs) + choice.set("maxOccurs", max_occurs) + choice.append(el) + + return namespaces, choice + @dataclass class Attribute: - name: str - type: str - required: str - default: int - description:str=None + element: ElementTree.Element - def to_etree_element(self): + def to_subtree(self): + namespaces = list() el = ElementTree.Element(_to_qname("xs:attribute")) - el.tail = "\n" - el.set("name", self.name) - el.set("type", self.type) - el.set("use", "required" if self.required == "1" else "optional") - - if self.default is not None: - el.set("default", self.default) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + + el.set("name", self.element.attrib["name"]) + + el_type = _to_simple_type(self.element.attrib["type"]) + el.set("type", el_type) + if el_type.startswith("types"): + namespaces.append("types") - return el + required = "required" if self.element.attrib["required"] == "1" else "optional" + el.set("use", required) + + if required == "optional" and "default" in self.element.attrib: + el.set("default", self.element.attrib["default"]) + + return namespaces, el @dataclass class ComplexType: - name: str element: ElementTree.Element + name: str = None - def to_etree_element(self, declared): + def to_subtree(self): + namespaces = list() elements = list() attributes = list() for attribute in self.element.findall("attribute"): - if "default" in attribute.attrib.keys(): - default = attribute.attrib["default"] - - attributes.append(Attribute( - name = attribute.attrib["name"], - type = attribute.attrib["type"], - required= attribute.attrib["required"], - default=default - ).to_etree_element()) + child_ns, child_el = Attribute(attribute).to_subtree() + attributes.append(child_el) + namespaces += child_ns for child in self.element.findall("element"): - if "copy_data" in child.attrib: - # special element for plugin.sdf to allow - # declare that any children are allowed - anyEl = ElementTree.Element(_to_qname("xs:any")) - anyEl.set("minOccurs", "0") - anyEl.set("maxOccurs", "unbounded") - anyEl.set("processContents", "skip") - elements.append(anyEl) - continue - - name = child.attrib["name"] - elements.append(declared["elements"][name].to_etree_element()) + child_ns, child_el = Element(child).to_subtree() + elements.append(child_el) + namespaces += child_ns for child in self.element.findall("include"): - child:ElementTree.Element - other_file = source_dir / child.attrib["filename"] - other_root = ElementTree.parse(other_file).getroot() - name = other_root.attrib["name"] - elements.append(declared["elements"][name].to_etree_element()) + child_ns, child_el = Include(child).to_subtree() + elements.append(child_el) + namespaces += child_ns el = ElementTree.Element(_to_qname("xs:complexType")) - el.set("name", self.name) - el.tail = "\n" + + if self.name: + el.set("name", self.name) if elements and "type" in self.element.attrib: - pass + raise NotImplementedError("Cant handle sub-elements for an element declaring a type.") elif "type" in self.element.attrib: + el_type = _to_simple_type(self.element.attrib["type"]) extension = ElementTree.Element(_to_qname("xs:extension")) - extension.set("base", self.element.attrib["type"]) + extension.set("base", el_type) + if el_type.startswith("types"): + namespaces.append("types") + extension.extend(attributes) simple_content = ElementTree.Element(_to_qname("xs:simpleContent")) simple_content.append(extension) el.append(simple_content) elif elements: choice = ElementTree.Element(_to_qname("xs:choice")) choice.set("maxOccurs", "unbounded") - choice.tail = "\n" choice.extend(elements) el.append(choice) - - el.extend(attributes) - - return el - - -@dataclass -class SimpleType: - name: str - namespace:str=None - - def __eq__(self, other): - return self.name == other.name - - def __hash__(self): - return hash(self.name) - - def to_etree_element(self): - name = self.name - known_types = { - "unsigned int": "xs:unsignedInt", - "unsigned long": "xs:unsignedLong", - "bool": "xs:boolean", - "string":"xs:string", - "double":"xs:double", - "int":"xs:int", - "float":"xs:float", - "char":"xs:char", - "vector3": "types:vector3", - "vector2d": "types:vector2d", - "vector2i": "types:vector2i", - "pose": "types:pose", - "time": "types:time", - "color": "types:color", - } - - try: - referred = known_types[name] - except KeyError: - raise RuntimeError(f"Unknown primitive type: {name}") from None - - restriction = ElementTree.Element(_to_qname("xs:restriction")) - restriction.set("base", referred) - - el = ElementTree.Element(_to_qname("xs:simpleType")) - el.set("name", name) - el.append(restriction) - el.tail = "\n" - - return el - -# parse file -# recurse elements of the file and track which xsd elements need to be generated -def expand(element:ElementTree.Element, declared:dict): - if element.tag == "description": - # not handled at this stage and may - # contain tags (see particle_emitter) - return - - for child in element: - expand(child, declared) - - desc_element = element.find("description") - description = desc_element.text if desc_element else None - - if element.tag == "element": - if "copy_data" in element.attrib: - # special element for plugin.sdf - # essentially allows any element to occur here - # we ignore it here, but insert while creating - # the pluginType - return - - name = element.attrib["name"] - element_type = name + "Type" - required = element.attrib["required"] - - num_children = len(element) - if element.find("description") is not None: - num_children = len(element) - 1 - - has_children = num_children > 0 - has_type = "type" in element.attrib - - if has_children and has_type: - # extens existing simple type - # Example: pose in pose.sdf - element_type = name + "Type" - declared.setdefault("simple_types", set()).add(SimpleType(element.attrib["type"])) - declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) - elif has_type: - # redefinition of simple type - # Example: (some) children of link.sdf - element_type = element.attrib["type"] - declared.setdefault("simple_types", set()).add(SimpleType(element_type)) - elif has_children: - # new complex type - # Example: world in world.sdf - element_type = name + "Type" - declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) + el.extend(attributes) else: - # element that wraps a string - # Example: audio_sink in audio_sink.sdf - element_type = name + "Type" - # declared.setdefault("simple_types", set()).add(SimpleType("string")) - declared.setdefault("complex_types", dict())[element_type] = ComplexType(name + "Type", element) - - - default = element.attrib["default"] if "default" in element.attrib else None - elements:dict = declared.setdefault("elements", dict()) - elements[name] = Element( - name, - type=element_type, - required=required, - default=default, - description=description - ) - elif element.tag == "include": - other_file = source_dir / element.attrib["filename"] - other_root = ElementTree.parse(other_file).getroot() - name = other_root.attrib["name"] - element_type = f"{other_file.stem}:{name}Type" - required = element.attrib["required"] - elements = declared.setdefault("elements", dict()) - elements[name] = Element(name, element_type, required) - declared.setdefault("imports", set()).add(other_file.stem) - elif element.tag == "attribute": - element_type = element.attrib["type"] - declared.setdefault("simple_types", set()).add(SimpleType(element_type)) - else: - raise RuntimeError(f"Unknown SDF element encountered: {element.tag}") + el.extend(attributes) -root = ElementTree.parse(source).getroot() -declared = dict() -declared.setdefault("imports", set()).add(source.stem) -expand(root, declared) + return namespaces, el + +xsd_schema = ElementTree.Element(_to_qname("xs:schema")) +xsd_schema.set("xmlns", namespaces[source.stem]) +xsd_schema.set("targetNamespace", namespaces[source.stem]) +# add types.xsd to every type schema +# TODO: only add it to those that use a types type +el = ElementTree.Element(_to_qname("xs:import")) +el.set("namespace", namespaces["types"]) +el.set("schemaLocation", f"./types.xsd") +el.tail = "\n" +xsd_schema.append(el) +xsd_schema.set(f"xmlns:types", namespaces["types"]) -xsd_root = ElementTree.Element(_to_qname("xs:schema")) -xsd_root.set("xmlns", namespaces[source.stem]) -xsd_root.set("targetNamespace", namespaces[source.stem]) -strings = dict() +used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() -for name in declared["imports"]: +for name in set(used_ns): el = ElementTree.Element(_to_qname("xs:import")) el.set("namespace", namespaces[name]) el.set("schemaLocation", f"./{name}Type.xsd") - el.tail = "\n" - xsd_root.append(el) - xsd_root.set(f"xmlns:{name}", namespaces[name]) - -if "simple_types" in declared: - el = ElementTree.Element(_to_qname("xs:import")) - el.set("namespace", namespaces["types"]) - el.set("schemaLocation", f"./types.xsd") - el.tail = "\n" - - xsd_root.append(el) - xsd_root.set(f"xmlns:types", namespaces["types"]) - - for simple_type in declared["simple_types"]: - xsd_root.append(simple_type.to_etree_element()) -else: - strings["simple_types"] = list() + xsd_schema.append(el) + xsd_schema.set(f"xmlns:{name}", namespaces[name]) -if "complex_types" in declared: - for complex_type in declared["complex_types"].values(): - xsd_root.append(complex_type.to_etree_element(declared)) -else: - strings["complex_types"] = list() +xsd_schema.append(element) out_dir = Path(args.target) @@ -348,19 +372,19 @@ def expand(element:ElementTree.Element, declared:dict): out_dir.mkdir(exist_ok=True, parents=True) # write type file with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: - ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") + ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") # write element file -xsd_root = ElementTree.Element(_to_qname("xs:schema")) -# xsd_root.set("targetNamespace", namespaces[source.stem]) -xsd_root.set("xmlns", namespaces[source.stem]) +xsd_schema = ElementTree.Element(_to_qname("xs:schema")) name = source.stem el = ElementTree.Element(_to_qname("xs:import")) el.set("namespace", namespaces[name]) -el.set("schemaLocation", f"./{name}.xsd") +el.set("schemaLocation", f"./{name}Type.xsd") el.tail = "\n" -xsd_root.append(el) -# xsd_root.set(f"xmlns:{name}", namespaces[name]) -xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) +xsd_schema.append(el) +xsd_schema.set(f"xmlns:{name}", namespaces[name]) +root_ns, root_el = Element(root).to_basic() +root_el.set("type", f"{name}:{root.attrib['name']}Type") +xsd_schema.append(root_el) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: - ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") + ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") From 1334ed1848958524a242fea52db8f46d9670469b Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 12:47:14 +0200 Subject: [PATCH 07/29] BUG: match time pattern with SDF default values --- sdf/1.8/schema/types.xsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdf/1.8/schema/types.xsd b/sdf/1.8/schema/types.xsd index e7c2e5f60..c706c1ab0 100644 --- a/sdf/1.8/schema/types.xsd +++ b/sdf/1.8/schema/types.xsd @@ -31,7 +31,8 @@ - + + From 2a86353c6d96530fae36f0946cf049559b6612bf Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 12:58:33 +0200 Subject: [PATCH 08/29] MAINT: refactor --- tools/xmlschema.py | 114 +++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 67 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 0906e00fd..383d21324 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -15,7 +15,6 @@ args = cmd_arg_parser.parse_args() source_dir = Path(args.directory) source:Path = source_dir / args.source -template_dir = Path(__file__).parent / "xsd_templates" root = ElementTree.parse(source).getroot() declared = dict() @@ -79,8 +78,9 @@ def to_subtree(self): desc_text = self.element.text if desc_text is None or desc_text == "": return list(), None - else: - desc_text = self.element.text.strip() + + desc_text = self.element.text.strip() + desc_text.replace("\n", " ") documentation = ElementTree.Element(_to_qname("xs:documentation")) documentation.text = desc_text @@ -109,37 +109,6 @@ def to_basic(self): el.append(doc_el) return namespaces, el - def to_etree_element(self): - namespaces = list() - el = ElementTree.Element(_to_qname("xs:element")) - el.set("name", self.name) - el.set("type", self.type) - if self.default: - el.set("default", self.default) - - docs = self.element.find("description") - if docs is not None: - doc_namespaces, doc_el = Description(docs).to_subtree() - namespaces.append(doc_namespaces) - if doc_el: - el.append(doc_el) - - - required_codes = { - "0" : ("0", "1"), - "1" : ("1", "1"), - "+" : ("1", "unbounded"), - "*" : ("0", "unbounded"), - "-1" : ("0", "0") - } - min_occurs, max_occurs = required_codes[self.element.attrib["required"]] - choice = ElementTree.Element(_to_qname("xs:choice")) - choice.set("minOccurs", min_occurs) - choice.set("maxOccurs", max_occurs) - choice.append(el) - - return choice - def to_subtree(self): namespaces = list() @@ -173,8 +142,23 @@ def to_subtree(self): num_children = len(self.element) - len(self.element.findall("description")) has_children = num_children > 0 has_type = "type" in self.element.attrib + has_ref = "ref" in self.element.attrib + + if has_ref: + # I couldn't quite work out how this tag is supposed to work + # it appears to refer to the file from which the root element + # should be used to define this element. This seems equivalent + # to though, so I am at a loss. + # This is a best guess implementation. + + other_file = source_dir / (self.element.attrib["ref"]+".sdf") + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + + child_ns, child_el = ComplexType(self.element).to_subtree() + el.set("type", f"{name}Type") - if has_type and has_children: + elif has_type and has_children: child_ns, child_el = ComplexType(self.element).to_subtree() namespaces.extend(child_ns) el.append(child_el) @@ -341,50 +325,46 @@ def to_subtree(self): return namespaces, el -xsd_schema = ElementTree.Element(_to_qname("xs:schema")) -xsd_schema.set("xmlns", namespaces[source.stem]) -xsd_schema.set("targetNamespace", namespaces[source.stem]) +def setup_schema(used_ns:list, use_default_ns:bool=True) -> ElementTree.Element: + xsd_schema = ElementTree.Element(_to_qname("xs:schema")) -# add types.xsd to every type schema -# TODO: only add it to those that use a types type -el = ElementTree.Element(_to_qname("xs:import")) -el.set("namespace", namespaces["types"]) -el.set("schemaLocation", f"./types.xsd") -el.tail = "\n" -xsd_schema.append(el) -xsd_schema.set(f"xmlns:types", namespaces["types"]) + if use_default_ns: + xsd_schema.set("xmlns", namespaces[source.stem]) + xsd_schema.set("targetNamespace", namespaces[source.stem]) -used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() + for name in set(used_ns): + el = ElementTree.Element(_to_qname("xs:import")) + el.set("namespace", namespaces[name]) -for name in set(used_ns): - el = ElementTree.Element(_to_qname("xs:import")) - el.set("namespace", namespaces[name]) - el.set("schemaLocation", f"./{name}Type.xsd") - - xsd_schema.append(el) - xsd_schema.set(f"xmlns:{name}", namespaces[name]) + if name == "types": + # types is a special class (not generated) + el.set("schemaLocation", f"./types.xsd") + else: + el.set("schemaLocation", f"./{name}Type.xsd") + + xsd_schema.append(el) + xsd_schema.set(f"xmlns:{name}", namespaces[name]) -xsd_schema.append(element) + return xsd_schema out_dir = Path(args.target) - if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) + # write type file +used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() +xsd_schema = setup_schema(used_ns) +xsd_schema.append(element) with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") # write element file -xsd_schema = ElementTree.Element(_to_qname("xs:schema")) -name = source.stem -el = ElementTree.Element(_to_qname("xs:import")) -el.set("namespace", namespaces[name]) -el.set("schemaLocation", f"./{name}Type.xsd") -el.tail = "\n" -xsd_schema.append(el) -xsd_schema.set(f"xmlns:{name}", namespaces[name]) -root_ns, root_el = Element(root).to_basic() -root_el.set("type", f"{name}:{root.attrib['name']}Type") -xsd_schema.append(root_el) +file_name = source.stem +tag_name = root.attrib["name"] +used_ns, element = Element(root).to_basic() +element.set("type", f"{file_name}:{tag_name}Type") +used_ns.append(file_name) +xsd_schema = setup_schema(used_ns, use_default_ns=False) +xsd_schema.append(element) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") From e36edce0161de87f2d36e66a761bee0b75e4052a Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Fri, 23 Jul 2021 20:44:56 +0200 Subject: [PATCH 09/29] BUG: missing sdf + copy types.xsd --- sdf/1.8/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdf/1.8/CMakeLists.txt b/sdf/1.8/CMakeLists.txt index d074806f5..5585aeb73 100644 --- a/sdf/1.8/CMakeLists.txt +++ b/sdf/1.8/CMakeLists.txt @@ -78,6 +78,8 @@ foreach(FIL ${sdfs}) VERBATIM) endforeach() +configure_file(schema/types.xsd ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + add_custom_target(schema1_8 ALL DEPENDS ${SDF_SCHEMA}) set_source_files_properties(${SDF_SCHEMA} PROPERTIES GENERATED TRUE) From 97cb10c6510ad22c203f731be79b1af70e8e99c1 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 25 Jul 2021 22:19:01 +0200 Subject: [PATCH 10/29] FEAT: python schema generator --- sdf/1.8/CMakeLists.txt | 2 +- sdf/1.8/schema/types.xsd | 2 +- tools/xmlschema.py | 349 +++++++++++++++++++ tools/xsd_templates/attribute.xsd | 2 + tools/xsd_templates/description.xsd | 5 + tools/xsd_templates/element.xsd | 3 + tools/xsd_templates/element_with_comment.xsd | 5 + tools/xsd_templates/expansion_type.xsd | 7 + tools/xsd_templates/file.xsd | 15 + tools/xsd_templates/import.xsd | 1 + tools/xsd_templates/include.xsd | 1 + tools/xsd_templates/type.xsd | 7 + 12 files changed, 397 insertions(+), 2 deletions(-) create mode 100644 tools/xmlschema.py create mode 100644 tools/xsd_templates/attribute.xsd create mode 100644 tools/xsd_templates/description.xsd create mode 100644 tools/xsd_templates/element.xsd create mode 100644 tools/xsd_templates/element_with_comment.xsd create mode 100644 tools/xsd_templates/expansion_type.xsd create mode 100644 tools/xsd_templates/file.xsd create mode 100644 tools/xsd_templates/import.xsd create mode 100644 tools/xsd_templates/include.xsd create mode 100644 tools/xsd_templates/type.xsd diff --git a/sdf/1.8/CMakeLists.txt b/sdf/1.8/CMakeLists.txt index 5585aeb73..530dcc83b 100644 --- a/sdf/1.8/CMakeLists.txt +++ b/sdf/1.8/CMakeLists.txt @@ -71,7 +71,7 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/tools/xmlschema.rb + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" diff --git a/sdf/1.8/schema/types.xsd b/sdf/1.8/schema/types.xsd index 53f3d8db6..e7c2e5f60 100644 --- a/sdf/1.8/schema/types.xsd +++ b/sdf/1.8/schema/types.xsd @@ -1,5 +1,5 @@ - + diff --git a/tools/xmlschema.py b/tools/xmlschema.py new file mode 100644 index 000000000..5974a6deb --- /dev/null +++ b/tools/xmlschema.py @@ -0,0 +1,349 @@ +from xml.etree import ElementTree +from pathlib import Path +import argparse +from dataclasses import dataclass +from typing import List + +def valid_path(arg:str): + path = Path(arg) + if path.exists(): + return path + else: + raise ValueError(f"The path does not exist: {arg}") + + +def existing_dir(arg:str): + path = valid_path(arg) + if path.is_dir(): + return path + else: + raise ValueError(f"Not a directory: {arg}") + +def existing_file(arg:str): + path = valid_path(arg) + if path.is_file(): + return path + else: + raise ValueError(f"Not a file: {arg}") + +def template_path(template_path:str): + if template_path == "": + return Path(__file__).parent / "xsd_templates" + else: + return existing_dir(template_path) + +cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") +cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=existing_file, help="SDF file to compile.") +cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=existing_dir, help="Directory containing all the SDF files.") +cmd_arg_parser.add_argument("-o", "--out", dest="target", default=".", type=str, help="Output directory for xsd file. Will be created if it doesn't exit.") +cmd_arg_parser.add_argument("--template-dir", dest="templates", default="", type=template_path, help="Location to search for xsd tempate files. Default: /tools/xsd_templates.") +cmd_arg_parser.add_argument("--ns-prefix", dest="ns_prefix", default="sdformat", type=str, help="Prefix for generated xsd namespaces.") + +args = cmd_arg_parser.parse_args() +source:Path = args.source +source_dir:Path = args.directory +xsd_file_template:str = (args.templates / "file.xsd").read_text() + + + +def _tabulate(input:str, offset:int=0) -> str: + formatted = "" + for line in input.split("\n")[:-1]: + formatted += (" " * 4) * offset + line + "\n" + return formatted + +# collect existing namespaces +namespaces = {"types": f"{args.ns_prefix}/types"} +for path in source_dir.iterdir(): + if not path.is_file(): + continue + + if not path.suffix == ".sdf": + continue + + namespaces[path.stem] = f"{args.ns_prefix}/{path.stem}" + +@dataclass +class Element: + name: str + type: str + required:str + default:str=None + description:str=None + + def to_xsd(self): + return f"" + + def to_typedef(self): + """The string used inside a ComplexType to refer to this element""" + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.required] + + if self.description: + template = (args.templates / "element_with_comment.xsd").read_text() + else: + template = (args.templates / "element.xsd").read_text() + template = template.format( + min_occurs=min_occurs, + max_occurs=max_occurs, + name = self.name, + type = self.type, + default = f"default='{self.default}'" if self.default is not None else "", + description = self.description, + ) + + return _tabulate(template, 2) + +@dataclass +class Attribute: + name: str + type: str + required: str + default: int + description:str=None + + def to_typedef(self): + template = (args.templates / "attribute.xsd").read_text() + template = template.format( + name = self.name, + type = self.type, + required = "required" if self.required == "1" else "optional", + default = self.default + ) + + return _tabulate(template, 1) + +@dataclass +class ComplexType: + name: str + element: ElementTree.Element + + def to_xsd(self, declared): + attributes = list() + elements = list() + + for attribute in self.element.findall("attribute"): + if "default" in attribute.attrib.keys(): + default = attribute.attrib["default"] + + attributes.append(Attribute( + name = attribute.attrib["name"], + type = attribute.attrib["type"], + required= attribute.attrib["required"], + default=default + ).to_typedef()) + + for child in self.element.findall("element"): + if "copy_data" in child.attrib: + # special element for plugin.sdf to allow + # declare that any children are allowed + any_xsd = "\n" + elements.append(_tabulate(any_xsd, 2)) + continue + + name = child.attrib["name"] + elements.append(declared["elements"][name].to_typedef()) + + for child in self.element.findall("include"): + child:ElementTree.Element + other_file = source_dir / child.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + elements.append(declared["elements"][name].to_typedef()) + + if "type" in self.element.attrib: + if len(elements) > 0: + raise RuntimeError("The compiler cant generate this type.") + + template = (args.templates / "expansion_type.xsd").read_text() + template = template.format( + name = self.name, + type = self.element.attrib["type"], + attributes = "\n".join(attributes) + ) + + else: + elements = "\n".join(elements) + attributes = "\n".join(attributes) + + template = (args.templates / "type.xsd").read_text() + template = template.format( + name = self.name, + elements = elements, + attributes = attributes + ) + + return template + +@dataclass +class SimpleType: + name: str + namespace:str=None + + def __eq__(self, other): + return self.name == other.name + + def __hash__(self): + return hash(self.name) + + def to_xsd(self): + name = self.name + known_types = { + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string":"xs:string", + "double":"xs:double", + "int":"xs:int", + "float":"xs:float", + "char":"xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } + + + try: + referred = known_types[name] + except KeyError: + raise RuntimeError(f"Unknown primitive type: {name}") from None + + return f"" + +# parse file +# recurse elements of the file and track which xsd elements need to be generated +def expand(element:ElementTree.Element, declared:dict): + if element.tag == "description": + # not handled at this stage and may + # contain tags (see particle_emitter) + return + + for child in element: + expand(child, declared) + + desc_element = element.find("description") + description = desc_element.text if desc_element else None + + if element.tag == "element": + if "copy_data" in element.attrib: + # special element for plugin.sdf + # essentially allows any element to occur here + # we ignore it here, but insert while creating + # the pluginType + return + + name = element.attrib["name"] + element_type = name + "Type" + required = element.attrib["required"] + + num_children = len(element) + if element.find("description") is not None: + num_children = len(element) - 1 + + has_children = num_children > 0 + has_type = "type" in element.attrib + + if has_children and has_type: + # extens existing simple type + # Example: pose in pose.sdf + element_type = name + "Type" + declared.setdefault("simple_types", set()).add(SimpleType(element.attrib["type"])) + declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) + elif has_type: + # redefinition of simple type + # Example: (some) children of link.sdf + element_type = element.attrib["type"] + declared.setdefault("simple_types", set()).add(SimpleType(element_type)) + elif has_children: + # new complex type + # Example: world in world.sdf + element_type = name + "Type" + declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) + else: + # element that wraps a string + # Example: audio_sink in audio_sink.sdf + element_type = name + "Type" + # declared.setdefault("simple_types", set()).add(SimpleType("string")) + declared.setdefault("complex_types", dict())[element_type] = ComplexType(name + "Type", element) + + + default = element.attrib["default"] if "default" in element.attrib else None + elements:dict = declared.setdefault("elements", dict()) + elements[name] = Element( + name, + type=element_type, + required=required, + default=default, + description=description + ) + elif element.tag == "include": + other_file = source_dir / element.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + element_type = f"{other_file.stem}:{name}Type" + required = element.attrib["required"] + elements = declared.setdefault("elements", dict()) + elements[name] = Element(name, element_type, required) + declared.setdefault("imports", set()).add(other_file.stem) + elif element.tag == "attribute": + element_type = element.attrib["type"] + declared.setdefault("simple_types", set()).add(SimpleType(element_type)) + else: + raise RuntimeError(f"Unknown SDF element encountered: {element.tag}") + +root = ElementTree.parse(source).getroot() +declared = dict() +declared.setdefault("imports", set()).add(source.stem) +expand(root, declared) + + +strings = dict() + +for name in declared["imports"]: + ns_elements = strings.setdefault("namespaces", []) + ns_elements.append(f" xmlns:{name}='{namespaces[name]}'") + imp_elements = strings.setdefault("imports", []) + imp_elements.append(f" ") + +if "simple_types" in declared: + for simple_type in declared["simple_types"]: + elements = strings.setdefault("simple_types", []) + elements.append(simple_type.to_xsd()) +else: + strings["simple_types"] = list() + +if "complex_types" in declared: + for complex_type in declared["complex_types"].values(): + elements = strings.setdefault("complex_types", []) + elements.append(complex_type.to_xsd(declared)) +else: + strings["complex_types"] = list() + + + +# write the file to disk +substitutions = { + "file_namespace": namespaces[source.stem], + "namespaces": "\n".join(strings["namespaces"]), + "imports":"\n".join(strings["imports"]), + "element":declared["elements"][root.attrib["name"]].to_xsd(), + "simple_types":"\n".join(strings["simple_types"]), + "complex_types":"\n".join(strings["complex_types"]) +} + +out_dir = Path(args.target) +if not out_dir.exists(): + out_dir.mkdir(exist_ok=True, parents=True) + +with open(out_dir / (source.stem + ".xsd"), "w") as out_file: + print(xsd_file_template.format(**substitutions), file=out_file) diff --git a/tools/xsd_templates/attribute.xsd b/tools/xsd_templates/attribute.xsd new file mode 100644 index 000000000..017d812f7 --- /dev/null +++ b/tools/xsd_templates/attribute.xsd @@ -0,0 +1,2 @@ + + diff --git a/tools/xsd_templates/description.xsd b/tools/xsd_templates/description.xsd new file mode 100644 index 000000000..23e703c7b --- /dev/null +++ b/tools/xsd_templates/description.xsd @@ -0,0 +1,5 @@ + + + <![CDATA[{INNER_TEXT}]]> + + diff --git a/tools/xsd_templates/element.xsd b/tools/xsd_templates/element.xsd new file mode 100644 index 000000000..701d170a6 --- /dev/null +++ b/tools/xsd_templates/element.xsd @@ -0,0 +1,3 @@ + + + diff --git a/tools/xsd_templates/element_with_comment.xsd b/tools/xsd_templates/element_with_comment.xsd new file mode 100644 index 000000000..a1526be89 --- /dev/null +++ b/tools/xsd_templates/element_with_comment.xsd @@ -0,0 +1,5 @@ + + + {description} + + diff --git a/tools/xsd_templates/expansion_type.xsd b/tools/xsd_templates/expansion_type.xsd new file mode 100644 index 000000000..31d60889a --- /dev/null +++ b/tools/xsd_templates/expansion_type.xsd @@ -0,0 +1,7 @@ + + + +{attributes} + + + diff --git a/tools/xsd_templates/file.xsd b/tools/xsd_templates/file.xsd new file mode 100644 index 000000000..f3660599c --- /dev/null +++ b/tools/xsd_templates/file.xsd @@ -0,0 +1,15 @@ + + + +{imports} + +{element} + +{simple_types} + +{complex_types} + + diff --git a/tools/xsd_templates/import.xsd b/tools/xsd_templates/import.xsd new file mode 100644 index 000000000..6bd2840b3 --- /dev/null +++ b/tools/xsd_templates/import.xsd @@ -0,0 +1 @@ + diff --git a/tools/xsd_templates/include.xsd b/tools/xsd_templates/include.xsd new file mode 100644 index 000000000..ed7359a6d --- /dev/null +++ b/tools/xsd_templates/include.xsd @@ -0,0 +1 @@ + diff --git a/tools/xsd_templates/type.xsd b/tools/xsd_templates/type.xsd new file mode 100644 index 000000000..691220cfa --- /dev/null +++ b/tools/xsd_templates/type.xsd @@ -0,0 +1,7 @@ + + +{elements} + + +{attributes} + From 688a9f7827f30b1e4c28f28b03c6a9b0c582fcb6 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 25 Jul 2021 22:29:47 +0200 Subject: [PATCH 11/29] MAINT: remove (unnecessary) validators --- sdf/1.8/CMakeLists.txt | 2 +- tools/xmlschema.py | 51 +++++++++--------------------------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/sdf/1.8/CMakeLists.txt b/sdf/1.8/CMakeLists.txt index 530dcc83b..e977a825f 100644 --- a/sdf/1.8/CMakeLists.txt +++ b/sdf/1.8/CMakeLists.txt @@ -72,7 +72,7 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py - ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} + ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" VERBATIM) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 5974a6deb..cb4307092 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -4,46 +4,17 @@ from dataclasses import dataclass from typing import List -def valid_path(arg:str): - path = Path(arg) - if path.exists(): - return path - else: - raise ValueError(f"The path does not exist: {arg}") - - -def existing_dir(arg:str): - path = valid_path(arg) - if path.is_dir(): - return path - else: - raise ValueError(f"Not a directory: {arg}") - -def existing_file(arg:str): - path = valid_path(arg) - if path.is_file(): - return path - else: - raise ValueError(f"Not a file: {arg}") - -def template_path(template_path:str): - if template_path == "": - return Path(__file__).parent / "xsd_templates" - else: - return existing_dir(template_path) - cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") -cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=existing_file, help="SDF file to compile.") -cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=existing_dir, help="Directory containing all the SDF files.") +cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") +cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") cmd_arg_parser.add_argument("-o", "--out", dest="target", default=".", type=str, help="Output directory for xsd file. Will be created if it doesn't exit.") -cmd_arg_parser.add_argument("--template-dir", dest="templates", default="", type=template_path, help="Location to search for xsd tempate files. Default: /tools/xsd_templates.") cmd_arg_parser.add_argument("--ns-prefix", dest="ns_prefix", default="sdformat", type=str, help="Prefix for generated xsd namespaces.") args = cmd_arg_parser.parse_args() -source:Path = args.source -source_dir:Path = args.directory -xsd_file_template:str = (args.templates / "file.xsd").read_text() - +source_dir = Path(args.directory) +source:Path = source_dir / args.source +template_dir = Path(__file__).parent / "xsd_templates" +xsd_file_template:str = (template_dir / "file.xsd").read_text() def _tabulate(input:str, offset:int=0) -> str: @@ -87,9 +58,9 @@ def to_typedef(self): min_occurs, max_occurs = required_codes[self.required] if self.description: - template = (args.templates / "element_with_comment.xsd").read_text() + template = (template_dir / "element_with_comment.xsd").read_text() else: - template = (args.templates / "element.xsd").read_text() + template = (template_dir / "element.xsd").read_text() template = template.format( min_occurs=min_occurs, max_occurs=max_occurs, @@ -110,7 +81,7 @@ class Attribute: description:str=None def to_typedef(self): - template = (args.templates / "attribute.xsd").read_text() + template = (template_dir / "attribute.xsd").read_text() template = template.format( name = self.name, type = self.type, @@ -162,7 +133,7 @@ def to_xsd(self, declared): if len(elements) > 0: raise RuntimeError("The compiler cant generate this type.") - template = (args.templates / "expansion_type.xsd").read_text() + template = (template_dir / "expansion_type.xsd").read_text() template = template.format( name = self.name, type = self.element.attrib["type"], @@ -173,7 +144,7 @@ def to_xsd(self, declared): elements = "\n".join(elements) attributes = "\n".join(attributes) - template = (args.templates / "type.xsd").read_text() + template = (template_dir / "type.xsd").read_text() template = template.format( name = self.name, elements = elements, From d1798cb51a645da6319cf504deb366a5121084d8 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Tue, 27 Jul 2021 20:13:44 +0200 Subject: [PATCH 12/29] MAINT: use etree for xml generation --- tools/xmlschema.py | 220 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 185 insertions(+), 35 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index cb4307092..c390d7899 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -1,9 +1,19 @@ from xml.etree import ElementTree + from pathlib import Path import argparse from dataclasses import dataclass from typing import List + +def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: + """Given XML/XSD input, add indentation and return it""" + formatted = "" + for line in input.split("\n")[:-1]: + formatted += style * offset + line + "\n" + return formatted + + cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") @@ -16,15 +26,11 @@ template_dir = Path(__file__).parent / "xsd_templates" xsd_file_template:str = (template_dir / "file.xsd").read_text() - -def _tabulate(input:str, offset:int=0) -> str: - formatted = "" - for line in input.split("\n")[:-1]: - formatted += (" " * 4) * offset + line + "\n" - return formatted - # collect existing namespaces -namespaces = {"types": f"{args.ns_prefix}/types"} +namespaces = { + "types": "http://sdformat.org/schemas/types.xsd", + "xs": "http://www.w3.org/2001/XMLSchema" +} for path in source_dir.iterdir(): if not path.is_file(): continue @@ -33,6 +39,17 @@ def _tabulate(input:str, offset:int=0) -> str: continue namespaces[path.stem] = f"{args.ns_prefix}/{path.stem}" + ElementTree.register_namespace(path.stem, f"{args.ns_prefix}/{path.stem}") + + +def _to_qname(name:str) -> str: + try: + prefix, name = name.split(":") + except ValueError: + # no prefix + return name + else: + return f"{{{namespaces[prefix]}}}{name}" @dataclass class Element: @@ -42,9 +59,6 @@ class Element: default:str=None description:str=None - def to_xsd(self): - return f"" - def to_typedef(self): """The string used inside a ComplexType to refer to this element""" @@ -70,7 +84,39 @@ def to_typedef(self): description = self.description, ) - return _tabulate(template, 2) + return template + + def to_basic(self): + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", self.name) + el.set("type", self.type) + if self.default: + el.set("default", self.default) + return el + + def to_etree_element(self): + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", self.name) + el.set("type", self.type) + if self.default: + el.set("default", self.default) + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.required] + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.tail = "\n" + choice.set("minOccurs", min_occurs) + choice.set("maxOccurs", max_occurs) + choice.append(el) + + return choice + @dataclass class Attribute: @@ -89,7 +135,20 @@ def to_typedef(self): default = self.default ) - return _tabulate(template, 1) + return template + + def to_etree_element(self): + el = ElementTree.Element(_to_qname("xs:attribute")) + el.tail = "\n" + + el.set("name", self.name) + el.set("type", self.type) + el.set("use", "required" if self.required == "1" else "optional") + + if self.default is not None: + el.set("default", self.default) + + return el @dataclass class ComplexType: @@ -104,12 +163,12 @@ def to_xsd(self, declared): if "default" in attribute.attrib.keys(): default = attribute.attrib["default"] - attributes.append(Attribute( + attributes.append(_tabulate(Attribute( name = attribute.attrib["name"], type = attribute.attrib["type"], required= attribute.attrib["required"], default=default - ).to_typedef()) + ).to_typedef(), 1)) for child in self.element.findall("element"): if "copy_data" in child.attrib: @@ -120,14 +179,14 @@ def to_xsd(self, declared): continue name = child.attrib["name"] - elements.append(declared["elements"][name].to_typedef()) + elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) for child in self.element.findall("include"): child:ElementTree.Element other_file = source_dir / child.attrib["filename"] other_root = ElementTree.parse(other_file).getroot() name = other_root.attrib["name"] - elements.append(declared["elements"][name].to_typedef()) + elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) if "type" in self.element.attrib: if len(elements) > 0: @@ -153,6 +212,58 @@ def to_xsd(self, declared): return template + def to_etree_element(self, declared): + elements = list() + attributes = list() + + for attribute in self.element.findall("attribute"): + if "default" in attribute.attrib.keys(): + default = attribute.attrib["default"] + + attributes.append(Attribute( + name = attribute.attrib["name"], + type = attribute.attrib["type"], + required= attribute.attrib["required"], + default=default + ).to_etree_element()) + + for child in self.element.findall("element"): + if "copy_data" in child.attrib: + # special element for plugin.sdf to allow + # declare that any children are allowed + anyEl = ElementTree.Element(_to_qname("xs:any")) + anyEl.set("minOccurs", "0") + anyEl.set("maxOccurs", "unbounded") + anyEl.set("processContents", "skip") + elements.append(anyEl) + continue + + name = child.attrib["name"] + elements.append(declared["elements"][name].to_etree_element()) + + for child in self.element.findall("include"): + child:ElementTree.Element + other_file = source_dir / child.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + elements.append(declared["elements"][name].to_etree_element()) + + el = ElementTree.Element(_to_qname("xs:complexType")) + el.set("name", self.name) + el.tail = "\n" + + if elements: + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.set("maxOccurs", "unbounded") + choice.tail = "\n" + choice.extend(elements) + el.append(choice) + + el.extend(attributes) + + return el + + @dataclass class SimpleType: name: str @@ -191,6 +302,40 @@ def to_xsd(self): return f"" + def to_etree_element(self): + name = self.name + known_types = { + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string":"xs:string", + "double":"xs:double", + "int":"xs:int", + "float":"xs:float", + "char":"xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } + + try: + referred = known_types[name] + except KeyError: + raise RuntimeError(f"Unknown primitive type: {name}") from None + + restriction = ElementTree.Element(_to_qname("xs:restriction")) + restriction.set("base", referred) + + el = ElementTree.Element(_to_qname("xs:simpleType")) + el.set("name", name) + el.append(restriction) + el.tail = "\n" + + return el + # parse file # recurse elements of the file and track which xsd elements need to be generated def expand(element:ElementTree.Element, declared:dict): @@ -278,43 +423,48 @@ def expand(element:ElementTree.Element, declared:dict): expand(root, declared) +xsd_root = ElementTree.Element(_to_qname("xs:schema")) +xsd_root.set("xmlns", namespaces[source.stem]) +xsd_root.set("targetNamespace", namespaces[source.stem]) +xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) strings = dict() for name in declared["imports"]: - ns_elements = strings.setdefault("namespaces", []) - ns_elements.append(f" xmlns:{name}='{namespaces[name]}'") - imp_elements = strings.setdefault("imports", []) - imp_elements.append(f" ") + el = ElementTree.Element(_to_qname("xs:import")) + el.set("namespace", namespaces[name]) + el.set("schemaLocation", f"./{name}.xsd") + el.tail = "\n" + + xsd_root.append(el) + xsd_root.set(f"xmlns:{name}", namespaces[name]) if "simple_types" in declared: + el = ElementTree.Element(_to_qname("xs:import")) + el.set("namespace", namespaces["types"]) + el.set("schemaLocation", f"./types.xsd") + el.tail = "\n" + + xsd_root.append(el) + xsd_root.set(f"xmlns:types", namespaces["types"]) + for simple_type in declared["simple_types"]: - elements = strings.setdefault("simple_types", []) - elements.append(simple_type.to_xsd()) + xsd_root.append(simple_type.to_etree_element()) else: strings["simple_types"] = list() if "complex_types" in declared: for complex_type in declared["complex_types"].values(): - elements = strings.setdefault("complex_types", []) - elements.append(complex_type.to_xsd(declared)) + xsd_root.append(complex_type.to_etree_element(declared)) else: strings["complex_types"] = list() # write the file to disk -substitutions = { - "file_namespace": namespaces[source.stem], - "namespaces": "\n".join(strings["namespaces"]), - "imports":"\n".join(strings["imports"]), - "element":declared["elements"][root.attrib["name"]].to_xsd(), - "simple_types":"\n".join(strings["simple_types"]), - "complex_types":"\n".join(strings["complex_types"]) -} - out_dir = Path(args.target) if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: - print(xsd_file_template.format(**substitutions), file=out_file) + ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") + # print(xsd_string, file=out_file) From b29e905df5b0d9ddbca8b282ec991a67e3cda643 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Tue, 27 Jul 2021 21:11:01 +0200 Subject: [PATCH 13/29] MAINT: remove refactored files --- tools/xmlschema.py | 156 ++++--------------- tools/xsd_templates/attribute.xsd | 2 - tools/xsd_templates/description.xsd | 5 - tools/xsd_templates/element.xsd | 3 - tools/xsd_templates/element_with_comment.xsd | 5 - tools/xsd_templates/expansion_type.xsd | 7 - tools/xsd_templates/file.xsd | 15 -- tools/xsd_templates/import.xsd | 1 - tools/xsd_templates/include.xsd | 1 - tools/xsd_templates/type.xsd | 7 - 10 files changed, 26 insertions(+), 176 deletions(-) delete mode 100644 tools/xsd_templates/attribute.xsd delete mode 100644 tools/xsd_templates/description.xsd delete mode 100644 tools/xsd_templates/element.xsd delete mode 100644 tools/xsd_templates/element_with_comment.xsd delete mode 100644 tools/xsd_templates/expansion_type.xsd delete mode 100644 tools/xsd_templates/file.xsd delete mode 100644 tools/xsd_templates/import.xsd delete mode 100644 tools/xsd_templates/include.xsd delete mode 100644 tools/xsd_templates/type.xsd diff --git a/tools/xmlschema.py b/tools/xmlschema.py index c390d7899..f97d09511 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -24,7 +24,6 @@ def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: source_dir = Path(args.directory) source:Path = source_dir / args.source template_dir = Path(__file__).parent / "xsd_templates" -xsd_file_template:str = (template_dir / "file.xsd").read_text() # collect existing namespaces namespaces = { @@ -59,33 +58,6 @@ class Element: default:str=None description:str=None - def to_typedef(self): - """The string used inside a ComplexType to refer to this element""" - - required_codes = { - "0" : ("0", "1"), - "1" : ("1", "1"), - "+" : ("1", "unbounded"), - "*" : ("0", "unbounded"), - "-1" : ("0", "0") - } - min_occurs, max_occurs = required_codes[self.required] - - if self.description: - template = (template_dir / "element_with_comment.xsd").read_text() - else: - template = (template_dir / "element.xsd").read_text() - template = template.format( - min_occurs=min_occurs, - max_occurs=max_occurs, - name = self.name, - type = self.type, - default = f"default='{self.default}'" if self.default is not None else "", - description = self.description, - ) - - return template - def to_basic(self): el = ElementTree.Element(_to_qname("xs:element")) el.set("name", self.name) @@ -126,17 +98,6 @@ class Attribute: default: int description:str=None - def to_typedef(self): - template = (template_dir / "attribute.xsd").read_text() - template = template.format( - name = self.name, - type = self.type, - required = "required" if self.required == "1" else "optional", - default = self.default - ) - - return template - def to_etree_element(self): el = ElementTree.Element(_to_qname("xs:attribute")) el.tail = "\n" @@ -155,63 +116,6 @@ class ComplexType: name: str element: ElementTree.Element - def to_xsd(self, declared): - attributes = list() - elements = list() - - for attribute in self.element.findall("attribute"): - if "default" in attribute.attrib.keys(): - default = attribute.attrib["default"] - - attributes.append(_tabulate(Attribute( - name = attribute.attrib["name"], - type = attribute.attrib["type"], - required= attribute.attrib["required"], - default=default - ).to_typedef(), 1)) - - for child in self.element.findall("element"): - if "copy_data" in child.attrib: - # special element for plugin.sdf to allow - # declare that any children are allowed - any_xsd = "\n" - elements.append(_tabulate(any_xsd, 2)) - continue - - name = child.attrib["name"] - elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) - - for child in self.element.findall("include"): - child:ElementTree.Element - other_file = source_dir / child.attrib["filename"] - other_root = ElementTree.parse(other_file).getroot() - name = other_root.attrib["name"] - elements.append(_tabulate(declared["elements"][name].to_typedef(), 2)) - - if "type" in self.element.attrib: - if len(elements) > 0: - raise RuntimeError("The compiler cant generate this type.") - - template = (template_dir / "expansion_type.xsd").read_text() - template = template.format( - name = self.name, - type = self.element.attrib["type"], - attributes = "\n".join(attributes) - ) - - else: - elements = "\n".join(elements) - attributes = "\n".join(attributes) - - template = (template_dir / "type.xsd").read_text() - template = template.format( - name = self.name, - elements = elements, - attributes = attributes - ) - - return template - def to_etree_element(self, declared): elements = list() attributes = list() @@ -252,7 +156,15 @@ def to_etree_element(self, declared): el.set("name", self.name) el.tail = "\n" - if elements: + if elements and "type" in self.element.attrib: + pass + elif "type" in self.element.attrib: + extension = ElementTree.Element(_to_qname("xs:extension")) + extension.set("base", self.element.attrib["type"]) + simple_content = ElementTree.Element(_to_qname("xs:simpleContent")) + simple_content.append(extension) + el.append(simple_content) + elif elements: choice = ElementTree.Element(_to_qname("xs:choice")) choice.set("maxOccurs", "unbounded") choice.tail = "\n" @@ -275,33 +187,6 @@ def __eq__(self, other): def __hash__(self): return hash(self.name) - def to_xsd(self): - name = self.name - known_types = { - "unsigned int": "xs:unsignedInt", - "unsigned long": "xs:unsignedLong", - "bool": "xs:boolean", - "string":"xs:string", - "double":"xs:double", - "int":"xs:int", - "float":"xs:float", - "char":"xs:char", - "vector3": "types:vector3", - "vector2d": "types:vector2d", - "vector2i": "types:vector2i", - "pose": "types:pose", - "time": "types:time", - "color": "types:color", - } - - - try: - referred = known_types[name] - except KeyError: - raise RuntimeError(f"Unknown primitive type: {name}") from None - - return f"" - def to_etree_element(self): name = self.name known_types = { @@ -426,13 +311,12 @@ def expand(element:ElementTree.Element, declared:dict): xsd_root = ElementTree.Element(_to_qname("xs:schema")) xsd_root.set("xmlns", namespaces[source.stem]) xsd_root.set("targetNamespace", namespaces[source.stem]) -xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) strings = dict() for name in declared["imports"]: el = ElementTree.Element(_to_qname("xs:import")) el.set("namespace", namespaces[name]) - el.set("schemaLocation", f"./{name}.xsd") + el.set("schemaLocation", f"./{name}Type.xsd") el.tail = "\n" xsd_root.append(el) @@ -458,13 +342,25 @@ def expand(element:ElementTree.Element, declared:dict): else: strings["complex_types"] = list() - - -# write the file to disk out_dir = Path(args.target) + if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) +# write type file +with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: + ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") +# write element file +xsd_root = ElementTree.Element(_to_qname("xs:schema")) +# xsd_root.set("targetNamespace", namespaces[source.stem]) +xsd_root.set("xmlns", namespaces[source.stem]) +name = source.stem +el = ElementTree.Element(_to_qname("xs:import")) +el.set("namespace", namespaces[name]) +el.set("schemaLocation", f"./{name}.xsd") +el.tail = "\n" +xsd_root.append(el) +# xsd_root.set(f"xmlns:{name}", namespaces[name]) +xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") - # print(xsd_string, file=out_file) diff --git a/tools/xsd_templates/attribute.xsd b/tools/xsd_templates/attribute.xsd deleted file mode 100644 index 017d812f7..000000000 --- a/tools/xsd_templates/attribute.xsd +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tools/xsd_templates/description.xsd b/tools/xsd_templates/description.xsd deleted file mode 100644 index 23e703c7b..000000000 --- a/tools/xsd_templates/description.xsd +++ /dev/null @@ -1,5 +0,0 @@ - - - <![CDATA[{INNER_TEXT}]]> - - diff --git a/tools/xsd_templates/element.xsd b/tools/xsd_templates/element.xsd deleted file mode 100644 index 701d170a6..000000000 --- a/tools/xsd_templates/element.xsd +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/tools/xsd_templates/element_with_comment.xsd b/tools/xsd_templates/element_with_comment.xsd deleted file mode 100644 index a1526be89..000000000 --- a/tools/xsd_templates/element_with_comment.xsd +++ /dev/null @@ -1,5 +0,0 @@ - - - {description} - - diff --git a/tools/xsd_templates/expansion_type.xsd b/tools/xsd_templates/expansion_type.xsd deleted file mode 100644 index 31d60889a..000000000 --- a/tools/xsd_templates/expansion_type.xsd +++ /dev/null @@ -1,7 +0,0 @@ - - - -{attributes} - - - diff --git a/tools/xsd_templates/file.xsd b/tools/xsd_templates/file.xsd deleted file mode 100644 index f3660599c..000000000 --- a/tools/xsd_templates/file.xsd +++ /dev/null @@ -1,15 +0,0 @@ - - - -{imports} - -{element} - -{simple_types} - -{complex_types} - - diff --git a/tools/xsd_templates/import.xsd b/tools/xsd_templates/import.xsd deleted file mode 100644 index 6bd2840b3..000000000 --- a/tools/xsd_templates/import.xsd +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/xsd_templates/include.xsd b/tools/xsd_templates/include.xsd deleted file mode 100644 index ed7359a6d..000000000 --- a/tools/xsd_templates/include.xsd +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/xsd_templates/type.xsd b/tools/xsd_templates/type.xsd deleted file mode 100644 index 691220cfa..000000000 --- a/tools/xsd_templates/type.xsd +++ /dev/null @@ -1,7 +0,0 @@ - - -{elements} - - -{attributes} - From bbec0be43b614193f6b393d7d85f3262fb3cd3c6 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 09:58:37 +0200 Subject: [PATCH 14/29] MAINT: integrate recursion into class structure --- tools/xmlschema.py | 498 ++++++++++++++++++++++++--------------------- 1 file changed, 261 insertions(+), 237 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index f97d09511..0906e00fd 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -6,14 +6,6 @@ from typing import List -def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: - """Given XML/XSD input, add indentation and return it""" - formatted = "" - for line in input.split("\n")[:-1]: - formatted += style * offset + line + "\n" - return formatted - - cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") @@ -25,6 +17,34 @@ def _tabulate(input:str, offset:int=0, style:str=" "*4) -> str: source:Path = source_dir / args.source template_dir = Path(__file__).parent / "xsd_templates" +root = ElementTree.parse(source).getroot() +declared = dict() +declared["imports"] = set() + + +def _to_simple_type(in_type:str) -> str: + known_types = { + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string":"xs:string", + "double":"xs:double", + "int":"xs:int", + "float":"xs:float", + "char":"xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } + + try: + return known_types[in_type] + except KeyError: + raise RuntimeError(f"Unknown simple type: {in_type}") from None + # collect existing namespaces namespaces = { "types": "http://sdformat.org/schemas/types.xsd", @@ -50,29 +70,61 @@ def _to_qname(name:str) -> str: else: return f"{{{namespaces[prefix]}}}{name}" + +@dataclass +class Description: + element: ElementTree.Element + + def to_subtree(self): + desc_text = self.element.text + if desc_text is None or desc_text == "": + return list(), None + else: + desc_text = self.element.text.strip() + + documentation = ElementTree.Element(_to_qname("xs:documentation")) + documentation.text = desc_text + documentation.set("xml:lang", "en") + annotation = ElementTree.Element(_to_qname("xs:annotation")) + annotation.append(documentation) + return list(), annotation + @dataclass class Element: - name: str - type: str - required:str - default:str=None - description:str=None + element: ElementTree.Element def to_basic(self): + namespaces = list() el = ElementTree.Element(_to_qname("xs:element")) - el.set("name", self.name) - el.set("type", self.type) - if self.default: - el.set("default", self.default) - return el + el.set("name", self.element.attrib["name"]) + if "type" in self.element.attrib: + el.set("type", self.element.attrib["type"]) + if "default" in self.element.attrib: + el.set("default", self.element.attrib["default"]) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + return namespaces, el def to_etree_element(self): + namespaces = list() el = ElementTree.Element(_to_qname("xs:element")) el.set("name", self.name) el.set("type", self.type) if self.default: el.set("default", self.default) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.append(doc_namespaces) + if doc_el: + el.append(doc_el) + + required_codes = { "0" : ("0", "1"), "1" : ("1", "1"), @@ -80,267 +132,239 @@ def to_etree_element(self): "*" : ("0", "unbounded"), "-1" : ("0", "0") } - min_occurs, max_occurs = required_codes[self.required] + min_occurs, max_occurs = required_codes[self.element.attrib["required"]] choice = ElementTree.Element(_to_qname("xs:choice")) - choice.tail = "\n" choice.set("minOccurs", min_occurs) choice.set("maxOccurs", max_occurs) choice.append(el) return choice + def to_subtree(self): + namespaces = list() + + if "copy_data" in self.element.attrib: + # special element for plugin.sdf to + # declare that any children are allowed + anyEl = ElementTree.Element(_to_qname("xs:any")) + anyEl.set("minOccurs", "0") + anyEl.set("maxOccurs", "unbounded") + anyEl.set("processContents", "skip") + + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + anyEl.append(doc_el) + + return namespaces, anyEl + + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", self.element.attrib["name"]) + + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + + num_children = len(self.element) - len(self.element.findall("description")) + has_children = num_children > 0 + has_type = "type" in self.element.attrib + + if has_type and has_children: + child_ns, child_el = ComplexType(self.element).to_subtree() + namespaces.extend(child_ns) + el.append(child_el) + elif has_children: + child_ns, child_el = ComplexType(self.element).to_subtree() + namespaces.extend(child_ns) + el.append(child_el) + elif has_type: + child_type = _to_simple_type(self.element.attrib["type"]) + el.set("type", child_type) + if child_type.startswith("types"): + namespaces.append("types") + else: + el.set("type", _to_simple_type("string")) + + if "default" in self.element.attrib: + el.set("default", self.element.attrib["default"]) + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.element.attrib["required"]] + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.set("minOccurs", min_occurs) + choice.set("maxOccurs", max_occurs) + choice.append(el) + + return namespaces, el + + +@dataclass +class Include: + element: ElementTree.Element + + def to_subtree(self): + namespaces = list() + other_file = source_dir / self.element.attrib["filename"] + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + + el = ElementTree.Element(_to_qname("xs:element")) + el.set("name", other_root.attrib["name"]) + + el.set("type", f"{other_file.stem}:{name}Type") + namespaces.append(f"{other_file.stem}") + + if "default" in other_root.attrib: + el.set("default", other_root.attrib["default"]) + + docs = other_root.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + + if self.element.attrib["required"] == "1": + el.set("type", f"{other_file.stem}:{name}Type") + + #TODO: remove this line + declared.setdefault("imports", set()).add(other_file.stem) + + + required_codes = { + "0" : ("0", "1"), + "1" : ("1", "1"), + "+" : ("1", "unbounded"), + "*" : ("0", "unbounded"), + "-1" : ("0", "0") + } + min_occurs, max_occurs = required_codes[self.element.attrib["required"]] + choice = ElementTree.Element(_to_qname("xs:choice")) + choice.set("minOccurs", min_occurs) + choice.set("maxOccurs", max_occurs) + choice.append(el) + + return namespaces, choice + @dataclass class Attribute: - name: str - type: str - required: str - default: int - description:str=None + element: ElementTree.Element - def to_etree_element(self): + def to_subtree(self): + namespaces = list() el = ElementTree.Element(_to_qname("xs:attribute")) - el.tail = "\n" - el.set("name", self.name) - el.set("type", self.type) - el.set("use", "required" if self.required == "1" else "optional") - - if self.default is not None: - el.set("default", self.default) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + + el.set("name", self.element.attrib["name"]) + + el_type = _to_simple_type(self.element.attrib["type"]) + el.set("type", el_type) + if el_type.startswith("types"): + namespaces.append("types") - return el + required = "required" if self.element.attrib["required"] == "1" else "optional" + el.set("use", required) + + if required == "optional" and "default" in self.element.attrib: + el.set("default", self.element.attrib["default"]) + + return namespaces, el @dataclass class ComplexType: - name: str element: ElementTree.Element + name: str = None - def to_etree_element(self, declared): + def to_subtree(self): + namespaces = list() elements = list() attributes = list() for attribute in self.element.findall("attribute"): - if "default" in attribute.attrib.keys(): - default = attribute.attrib["default"] - - attributes.append(Attribute( - name = attribute.attrib["name"], - type = attribute.attrib["type"], - required= attribute.attrib["required"], - default=default - ).to_etree_element()) + child_ns, child_el = Attribute(attribute).to_subtree() + attributes.append(child_el) + namespaces += child_ns for child in self.element.findall("element"): - if "copy_data" in child.attrib: - # special element for plugin.sdf to allow - # declare that any children are allowed - anyEl = ElementTree.Element(_to_qname("xs:any")) - anyEl.set("minOccurs", "0") - anyEl.set("maxOccurs", "unbounded") - anyEl.set("processContents", "skip") - elements.append(anyEl) - continue - - name = child.attrib["name"] - elements.append(declared["elements"][name].to_etree_element()) + child_ns, child_el = Element(child).to_subtree() + elements.append(child_el) + namespaces += child_ns for child in self.element.findall("include"): - child:ElementTree.Element - other_file = source_dir / child.attrib["filename"] - other_root = ElementTree.parse(other_file).getroot() - name = other_root.attrib["name"] - elements.append(declared["elements"][name].to_etree_element()) + child_ns, child_el = Include(child).to_subtree() + elements.append(child_el) + namespaces += child_ns el = ElementTree.Element(_to_qname("xs:complexType")) - el.set("name", self.name) - el.tail = "\n" + + if self.name: + el.set("name", self.name) if elements and "type" in self.element.attrib: - pass + raise NotImplementedError("Cant handle sub-elements for an element declaring a type.") elif "type" in self.element.attrib: + el_type = _to_simple_type(self.element.attrib["type"]) extension = ElementTree.Element(_to_qname("xs:extension")) - extension.set("base", self.element.attrib["type"]) + extension.set("base", el_type) + if el_type.startswith("types"): + namespaces.append("types") + extension.extend(attributes) simple_content = ElementTree.Element(_to_qname("xs:simpleContent")) simple_content.append(extension) el.append(simple_content) elif elements: choice = ElementTree.Element(_to_qname("xs:choice")) choice.set("maxOccurs", "unbounded") - choice.tail = "\n" choice.extend(elements) el.append(choice) - - el.extend(attributes) - - return el - - -@dataclass -class SimpleType: - name: str - namespace:str=None - - def __eq__(self, other): - return self.name == other.name - - def __hash__(self): - return hash(self.name) - - def to_etree_element(self): - name = self.name - known_types = { - "unsigned int": "xs:unsignedInt", - "unsigned long": "xs:unsignedLong", - "bool": "xs:boolean", - "string":"xs:string", - "double":"xs:double", - "int":"xs:int", - "float":"xs:float", - "char":"xs:char", - "vector3": "types:vector3", - "vector2d": "types:vector2d", - "vector2i": "types:vector2i", - "pose": "types:pose", - "time": "types:time", - "color": "types:color", - } - - try: - referred = known_types[name] - except KeyError: - raise RuntimeError(f"Unknown primitive type: {name}") from None - - restriction = ElementTree.Element(_to_qname("xs:restriction")) - restriction.set("base", referred) - - el = ElementTree.Element(_to_qname("xs:simpleType")) - el.set("name", name) - el.append(restriction) - el.tail = "\n" - - return el - -# parse file -# recurse elements of the file and track which xsd elements need to be generated -def expand(element:ElementTree.Element, declared:dict): - if element.tag == "description": - # not handled at this stage and may - # contain tags (see particle_emitter) - return - - for child in element: - expand(child, declared) - - desc_element = element.find("description") - description = desc_element.text if desc_element else None - - if element.tag == "element": - if "copy_data" in element.attrib: - # special element for plugin.sdf - # essentially allows any element to occur here - # we ignore it here, but insert while creating - # the pluginType - return - - name = element.attrib["name"] - element_type = name + "Type" - required = element.attrib["required"] - - num_children = len(element) - if element.find("description") is not None: - num_children = len(element) - 1 - - has_children = num_children > 0 - has_type = "type" in element.attrib - - if has_children and has_type: - # extens existing simple type - # Example: pose in pose.sdf - element_type = name + "Type" - declared.setdefault("simple_types", set()).add(SimpleType(element.attrib["type"])) - declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) - elif has_type: - # redefinition of simple type - # Example: (some) children of link.sdf - element_type = element.attrib["type"] - declared.setdefault("simple_types", set()).add(SimpleType(element_type)) - elif has_children: - # new complex type - # Example: world in world.sdf - element_type = name + "Type" - declared.setdefault("complex_types", dict())[element_type] = ComplexType(element_type, element) + el.extend(attributes) else: - # element that wraps a string - # Example: audio_sink in audio_sink.sdf - element_type = name + "Type" - # declared.setdefault("simple_types", set()).add(SimpleType("string")) - declared.setdefault("complex_types", dict())[element_type] = ComplexType(name + "Type", element) - - - default = element.attrib["default"] if "default" in element.attrib else None - elements:dict = declared.setdefault("elements", dict()) - elements[name] = Element( - name, - type=element_type, - required=required, - default=default, - description=description - ) - elif element.tag == "include": - other_file = source_dir / element.attrib["filename"] - other_root = ElementTree.parse(other_file).getroot() - name = other_root.attrib["name"] - element_type = f"{other_file.stem}:{name}Type" - required = element.attrib["required"] - elements = declared.setdefault("elements", dict()) - elements[name] = Element(name, element_type, required) - declared.setdefault("imports", set()).add(other_file.stem) - elif element.tag == "attribute": - element_type = element.attrib["type"] - declared.setdefault("simple_types", set()).add(SimpleType(element_type)) - else: - raise RuntimeError(f"Unknown SDF element encountered: {element.tag}") + el.extend(attributes) -root = ElementTree.parse(source).getroot() -declared = dict() -declared.setdefault("imports", set()).add(source.stem) -expand(root, declared) + return namespaces, el + +xsd_schema = ElementTree.Element(_to_qname("xs:schema")) +xsd_schema.set("xmlns", namespaces[source.stem]) +xsd_schema.set("targetNamespace", namespaces[source.stem]) +# add types.xsd to every type schema +# TODO: only add it to those that use a types type +el = ElementTree.Element(_to_qname("xs:import")) +el.set("namespace", namespaces["types"]) +el.set("schemaLocation", f"./types.xsd") +el.tail = "\n" +xsd_schema.append(el) +xsd_schema.set(f"xmlns:types", namespaces["types"]) -xsd_root = ElementTree.Element(_to_qname("xs:schema")) -xsd_root.set("xmlns", namespaces[source.stem]) -xsd_root.set("targetNamespace", namespaces[source.stem]) -strings = dict() +used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() -for name in declared["imports"]: +for name in set(used_ns): el = ElementTree.Element(_to_qname("xs:import")) el.set("namespace", namespaces[name]) el.set("schemaLocation", f"./{name}Type.xsd") - el.tail = "\n" - xsd_root.append(el) - xsd_root.set(f"xmlns:{name}", namespaces[name]) - -if "simple_types" in declared: - el = ElementTree.Element(_to_qname("xs:import")) - el.set("namespace", namespaces["types"]) - el.set("schemaLocation", f"./types.xsd") - el.tail = "\n" - - xsd_root.append(el) - xsd_root.set(f"xmlns:types", namespaces["types"]) - - for simple_type in declared["simple_types"]: - xsd_root.append(simple_type.to_etree_element()) -else: - strings["simple_types"] = list() + xsd_schema.append(el) + xsd_schema.set(f"xmlns:{name}", namespaces[name]) -if "complex_types" in declared: - for complex_type in declared["complex_types"].values(): - xsd_root.append(complex_type.to_etree_element(declared)) -else: - strings["complex_types"] = list() +xsd_schema.append(element) out_dir = Path(args.target) @@ -348,19 +372,19 @@ def expand(element:ElementTree.Element, declared:dict): out_dir.mkdir(exist_ok=True, parents=True) # write type file with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: - ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") + ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") # write element file -xsd_root = ElementTree.Element(_to_qname("xs:schema")) -# xsd_root.set("targetNamespace", namespaces[source.stem]) -xsd_root.set("xmlns", namespaces[source.stem]) +xsd_schema = ElementTree.Element(_to_qname("xs:schema")) name = source.stem el = ElementTree.Element(_to_qname("xs:import")) el.set("namespace", namespaces[name]) -el.set("schemaLocation", f"./{name}.xsd") +el.set("schemaLocation", f"./{name}Type.xsd") el.tail = "\n" -xsd_root.append(el) -# xsd_root.set(f"xmlns:{name}", namespaces[name]) -xsd_root.append(declared["elements"][root.attrib["name"]].to_basic()) +xsd_schema.append(el) +xsd_schema.set(f"xmlns:{name}", namespaces[name]) +root_ns, root_el = Element(root).to_basic() +root_el.set("type", f"{name}:{root.attrib['name']}Type") +xsd_schema.append(root_el) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: - ElementTree.ElementTree(xsd_root).write(out_file, encoding="unicode") + ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") From d856458900f76e79721910d7ccae613ddfc08441 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 12:47:14 +0200 Subject: [PATCH 15/29] BUG: match time pattern with SDF default values --- sdf/1.8/schema/types.xsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdf/1.8/schema/types.xsd b/sdf/1.8/schema/types.xsd index e7c2e5f60..c706c1ab0 100644 --- a/sdf/1.8/schema/types.xsd +++ b/sdf/1.8/schema/types.xsd @@ -31,7 +31,8 @@ - + + From 887ef676b3f5818c86e9defe6e8e2bfdccd90b14 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 12:58:33 +0200 Subject: [PATCH 16/29] MAINT: refactor --- tools/xmlschema.py | 114 +++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 67 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 0906e00fd..383d21324 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -15,7 +15,6 @@ args = cmd_arg_parser.parse_args() source_dir = Path(args.directory) source:Path = source_dir / args.source -template_dir = Path(__file__).parent / "xsd_templates" root = ElementTree.parse(source).getroot() declared = dict() @@ -79,8 +78,9 @@ def to_subtree(self): desc_text = self.element.text if desc_text is None or desc_text == "": return list(), None - else: - desc_text = self.element.text.strip() + + desc_text = self.element.text.strip() + desc_text.replace("\n", " ") documentation = ElementTree.Element(_to_qname("xs:documentation")) documentation.text = desc_text @@ -109,37 +109,6 @@ def to_basic(self): el.append(doc_el) return namespaces, el - def to_etree_element(self): - namespaces = list() - el = ElementTree.Element(_to_qname("xs:element")) - el.set("name", self.name) - el.set("type", self.type) - if self.default: - el.set("default", self.default) - - docs = self.element.find("description") - if docs is not None: - doc_namespaces, doc_el = Description(docs).to_subtree() - namespaces.append(doc_namespaces) - if doc_el: - el.append(doc_el) - - - required_codes = { - "0" : ("0", "1"), - "1" : ("1", "1"), - "+" : ("1", "unbounded"), - "*" : ("0", "unbounded"), - "-1" : ("0", "0") - } - min_occurs, max_occurs = required_codes[self.element.attrib["required"]] - choice = ElementTree.Element(_to_qname("xs:choice")) - choice.set("minOccurs", min_occurs) - choice.set("maxOccurs", max_occurs) - choice.append(el) - - return choice - def to_subtree(self): namespaces = list() @@ -173,8 +142,23 @@ def to_subtree(self): num_children = len(self.element) - len(self.element.findall("description")) has_children = num_children > 0 has_type = "type" in self.element.attrib + has_ref = "ref" in self.element.attrib + + if has_ref: + # I couldn't quite work out how this tag is supposed to work + # it appears to refer to the file from which the root element + # should be used to define this element. This seems equivalent + # to though, so I am at a loss. + # This is a best guess implementation. + + other_file = source_dir / (self.element.attrib["ref"]+".sdf") + other_root = ElementTree.parse(other_file).getroot() + name = other_root.attrib["name"] + + child_ns, child_el = ComplexType(self.element).to_subtree() + el.set("type", f"{name}Type") - if has_type and has_children: + elif has_type and has_children: child_ns, child_el = ComplexType(self.element).to_subtree() namespaces.extend(child_ns) el.append(child_el) @@ -341,50 +325,46 @@ def to_subtree(self): return namespaces, el -xsd_schema = ElementTree.Element(_to_qname("xs:schema")) -xsd_schema.set("xmlns", namespaces[source.stem]) -xsd_schema.set("targetNamespace", namespaces[source.stem]) +def setup_schema(used_ns:list, use_default_ns:bool=True) -> ElementTree.Element: + xsd_schema = ElementTree.Element(_to_qname("xs:schema")) -# add types.xsd to every type schema -# TODO: only add it to those that use a types type -el = ElementTree.Element(_to_qname("xs:import")) -el.set("namespace", namespaces["types"]) -el.set("schemaLocation", f"./types.xsd") -el.tail = "\n" -xsd_schema.append(el) -xsd_schema.set(f"xmlns:types", namespaces["types"]) + if use_default_ns: + xsd_schema.set("xmlns", namespaces[source.stem]) + xsd_schema.set("targetNamespace", namespaces[source.stem]) -used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() + for name in set(used_ns): + el = ElementTree.Element(_to_qname("xs:import")) + el.set("namespace", namespaces[name]) -for name in set(used_ns): - el = ElementTree.Element(_to_qname("xs:import")) - el.set("namespace", namespaces[name]) - el.set("schemaLocation", f"./{name}Type.xsd") - - xsd_schema.append(el) - xsd_schema.set(f"xmlns:{name}", namespaces[name]) + if name == "types": + # types is a special class (not generated) + el.set("schemaLocation", f"./types.xsd") + else: + el.set("schemaLocation", f"./{name}Type.xsd") + + xsd_schema.append(el) + xsd_schema.set(f"xmlns:{name}", namespaces[name]) -xsd_schema.append(element) + return xsd_schema out_dir = Path(args.target) - if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) + # write type file +used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() +xsd_schema = setup_schema(used_ns) +xsd_schema.append(element) with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") # write element file -xsd_schema = ElementTree.Element(_to_qname("xs:schema")) -name = source.stem -el = ElementTree.Element(_to_qname("xs:import")) -el.set("namespace", namespaces[name]) -el.set("schemaLocation", f"./{name}Type.xsd") -el.tail = "\n" -xsd_schema.append(el) -xsd_schema.set(f"xmlns:{name}", namespaces[name]) -root_ns, root_el = Element(root).to_basic() -root_el.set("type", f"{name}:{root.attrib['name']}Type") -xsd_schema.append(root_el) +file_name = source.stem +tag_name = root.attrib["name"] +used_ns, element = Element(root).to_basic() +element.set("type", f"{file_name}:{tag_name}Type") +used_ns.append(file_name) +xsd_schema = setup_schema(used_ns, use_default_ns=False) +xsd_schema.append(element) with open(out_dir / (source.stem + ".xsd"), "w") as out_file: ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") From 587d6527c558249c694d2cca52468cd1fea8f10e Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Wed, 28 Jul 2021 22:24:37 +0200 Subject: [PATCH 17/29] MAINT: migrate v1.5 - v1.7 to new XSD parser --- sdf/1.5/CMakeLists.txt | 6 ++++-- sdf/1.6/CMakeLists.txt | 7 +++++-- sdf/1.7/CMakeLists.txt | 6 ++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sdf/1.5/CMakeLists.txt b/sdf/1.5/CMakeLists.txt index 60fd6a15a..73dca8c0f 100644 --- a/sdf/1.5/CMakeLists.txt +++ b/sdf/1.5/CMakeLists.txt @@ -65,13 +65,15 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/tools/xmlschema.rb - ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py + ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" VERBATIM) endforeach() +configure_file(schema/types.xsd ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + add_custom_target(schema1_5 ALL DEPENDS ${SDF_SCHEMA}) set_source_files_properties(${SDF_SCHEMA} PROPERTIES GENERATED TRUE) diff --git a/sdf/1.6/CMakeLists.txt b/sdf/1.6/CMakeLists.txt index fe972e4c5..496b9cad7 100644 --- a/sdf/1.6/CMakeLists.txt +++ b/sdf/1.6/CMakeLists.txt @@ -69,13 +69,16 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/tools/xmlschema.rb - ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py + ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" VERBATIM) endforeach() +configure_file(schema/types.xsd ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + + add_custom_target(schema1_6 ALL DEPENDS ${SDF_SCHEMA}) set_source_files_properties(${SDF_SCHEMA} PROPERTIES GENERATED TRUE) diff --git a/sdf/1.7/CMakeLists.txt b/sdf/1.7/CMakeLists.txt index 516a7874b..3fa86f5a2 100644 --- a/sdf/1.7/CMakeLists.txt +++ b/sdf/1.7/CMakeLists.txt @@ -69,13 +69,15 @@ foreach(FIL ${sdfs}) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.xsd" - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/tools/xmlschema.rb - ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${ABS_FIL} -o ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/xmlschema.py + ARGS -s ${CMAKE_CURRENT_SOURCE_DIR} -i ${FIL} -o ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${ABS_FIL} COMMENT "Running xml schema compiler on ${FIL}" VERBATIM) endforeach() +configure_file(schema/types.xsd ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + add_custom_target(schema1_7 ALL DEPENDS ${SDF_SCHEMA}) set_source_files_properties(${SDF_SCHEMA} PROPERTIES GENERATED TRUE) From 38d5337f9beffb8adfc469ddc0c4600e003e4ed7 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Thu, 29 Jul 2021 14:17:20 +0200 Subject: [PATCH 18/29] BUG: backport changes of types.xsd --- sdf/1.5/schema/types.xsd | 5 +++-- sdf/1.6/schema/types.xsd | 5 +++-- sdf/1.7/schema/types.xsd | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sdf/1.5/schema/types.xsd b/sdf/1.5/schema/types.xsd index 53f3d8db6..c706c1ab0 100644 --- a/sdf/1.5/schema/types.xsd +++ b/sdf/1.5/schema/types.xsd @@ -1,5 +1,5 @@ - + @@ -31,7 +31,8 @@ - + + diff --git a/sdf/1.6/schema/types.xsd b/sdf/1.6/schema/types.xsd index 53f3d8db6..c706c1ab0 100644 --- a/sdf/1.6/schema/types.xsd +++ b/sdf/1.6/schema/types.xsd @@ -1,5 +1,5 @@ - + @@ -31,7 +31,8 @@ - + + diff --git a/sdf/1.7/schema/types.xsd b/sdf/1.7/schema/types.xsd index 53f3d8db6..c706c1ab0 100644 --- a/sdf/1.7/schema/types.xsd +++ b/sdf/1.7/schema/types.xsd @@ -1,5 +1,5 @@ - + @@ -31,7 +31,8 @@ - + + From 290da7beb25a086fc4e17e8c6bb049f732676506 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 12:49:29 +0200 Subject: [PATCH 19/29] MAINT: move description into type schema --- tools/xmlschema.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 383d21324..911e1d0b3 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -101,12 +101,7 @@ def to_basic(self): el.set("type", self.element.attrib["type"]) if "default" in self.element.attrib: el.set("default", self.element.attrib["default"]) - docs = self.element.find("description") - if docs is not None: - doc_namespaces, doc_el = Description(docs).to_subtree() - namespaces.extend(doc_namespaces) - if doc_el: - el.append(doc_el) + return namespaces, el def to_subtree(self): @@ -302,6 +297,13 @@ def to_subtree(self): if self.name: el.set("name", self.name) + docs = self.element.find("description") + if docs is not None: + doc_namespaces, doc_el = Description(docs).to_subtree() + namespaces.extend(doc_namespaces) + if doc_el: + el.append(doc_el) + if elements and "type" in self.element.attrib: raise NotImplementedError("Cant handle sub-elements for an element declaring a type.") elif "type" in self.element.attrib: From a666c1e00d66eeae02aab2ad62e98345e0d94b70 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 14:28:54 +0200 Subject: [PATCH 20/29] MAINT: unwrap elements from choice block The original script wrapped every element inside a choice block; however - as this is only a single element - the choice block has essentially no effect. --- tools/xmlschema.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 911e1d0b3..9d6742f3b 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -180,10 +180,8 @@ def to_subtree(self): "-1" : ("0", "0") } min_occurs, max_occurs = required_codes[self.element.attrib["required"]] - choice = ElementTree.Element(_to_qname("xs:choice")) - choice.set("minOccurs", min_occurs) - choice.set("maxOccurs", max_occurs) - choice.append(el) + el.set("minOccurs", min_occurs) + el.set("maxOccurs", max_occurs) return namespaces, el @@ -229,12 +227,10 @@ def to_subtree(self): "-1" : ("0", "0") } min_occurs, max_occurs = required_codes[self.element.attrib["required"]] - choice = ElementTree.Element(_to_qname("xs:choice")) - choice.set("minOccurs", min_occurs) - choice.set("maxOccurs", max_occurs) - choice.append(el) + el.set("minOccurs", min_occurs) + el.set("maxOccurs", max_occurs) - return namespaces, choice + return namespaces, el @dataclass From e19c366ffd2af6e9a8ebf2839cd4255ba40523ec Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 14:57:41 +0200 Subject: [PATCH 21/29] BUG: partially fix ignored min/maxOccurs --- tools/xmlschema.py | 150 ++++++++++++++++++++++++++++++--------------- 1 file changed, 101 insertions(+), 49 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 9d6742f3b..ac9488851 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -6,48 +6,78 @@ from typing import List -cmd_arg_parser = argparse.ArgumentParser(description="Create an XML schema from a SDFormat file.") -cmd_arg_parser.add_argument("-i", "--in", dest="source", required=True, type=str, help="SDF file inside of directory to compile.") -cmd_arg_parser.add_argument("-s", "--sdf", dest="directory", required=True, type=str, help="Directory containing all the SDF files.") -cmd_arg_parser.add_argument("-o", "--out", dest="target", default=".", type=str, help="Output directory for xsd file. Will be created if it doesn't exit.") -cmd_arg_parser.add_argument("--ns-prefix", dest="ns_prefix", default="sdformat", type=str, help="Prefix for generated xsd namespaces.") +cmd_arg_parser = argparse.ArgumentParser( + description="Create an XML schema from a SDFormat file." +) +cmd_arg_parser.add_argument( + "-i", + "--in", + dest="source", + required=True, + type=str, + help="SDF file inside of directory to compile.", +) +cmd_arg_parser.add_argument( + "-s", + "--sdf", + dest="directory", + required=True, + type=str, + help="Directory containing all the SDF files.", +) +cmd_arg_parser.add_argument( + "-o", + "--out", + dest="target", + default=".", + type=str, + help="Output directory for xsd file. Will be created if it doesn't exit.", +) +cmd_arg_parser.add_argument( + "--ns-prefix", + dest="ns_prefix", + default="sdformat", + type=str, + help="Prefix for generated xsd namespaces.", +) args = cmd_arg_parser.parse_args() source_dir = Path(args.directory) -source:Path = source_dir / args.source +source: Path = source_dir / args.source root = ElementTree.parse(source).getroot() declared = dict() declared["imports"] = set() -def _to_simple_type(in_type:str) -> str: +def _to_simple_type(in_type: str) -> str: known_types = { - "unsigned int": "xs:unsignedInt", - "unsigned long": "xs:unsignedLong", - "bool": "xs:boolean", - "string":"xs:string", - "double":"xs:double", - "int":"xs:int", - "float":"xs:float", - "char":"xs:char", - "vector3": "types:vector3", - "vector2d": "types:vector2d", - "vector2i": "types:vector2i", - "pose": "types:pose", - "time": "types:time", - "color": "types:color", - } + "unsigned int": "xs:unsignedInt", + "unsigned long": "xs:unsignedLong", + "bool": "xs:boolean", + "string": "xs:string", + "double": "xs:double", + "int": "xs:int", + "float": "xs:float", + "char": "xs:char", + "vector3": "types:vector3", + "vector2d": "types:vector2d", + "vector2i": "types:vector2i", + "pose": "types:pose", + "time": "types:time", + "color": "types:color", + } try: return known_types[in_type] except KeyError: raise RuntimeError(f"Unknown simple type: {in_type}") from None + # collect existing namespaces namespaces = { "types": "http://sdformat.org/schemas/types.xsd", - "xs": "http://www.w3.org/2001/XMLSchema" + "xs": "http://www.w3.org/2001/XMLSchema", } for path in source_dir.iterdir(): if not path.is_file(): @@ -60,7 +90,7 @@ def _to_simple_type(in_type:str) -> str: ElementTree.register_namespace(path.stem, f"{args.ns_prefix}/{path.stem}") -def _to_qname(name:str) -> str: +def _to_qname(name: str) -> str: try: prefix, name = name.split(":") except ValueError: @@ -89,6 +119,7 @@ def to_subtree(self): annotation.append(documentation) return list(), annotation + @dataclass class Element: element: ElementTree.Element @@ -114,7 +145,7 @@ def to_subtree(self): anyEl.set("minOccurs", "0") anyEl.set("maxOccurs", "unbounded") anyEl.set("processContents", "skip") - + docs = self.element.find("description") if docs is not None: doc_namespaces, doc_el = Description(docs).to_subtree() @@ -146,7 +177,7 @@ def to_subtree(self): # to though, so I am at a loss. # This is a best guess implementation. - other_file = source_dir / (self.element.attrib["ref"]+".sdf") + other_file = source_dir / (self.element.attrib["ref"] + ".sdf") other_root = ElementTree.parse(other_file).getroot() name = other_root.attrib["name"] @@ -168,16 +199,16 @@ def to_subtree(self): namespaces.append("types") else: el.set("type", _to_simple_type("string")) - + if "default" in self.element.attrib: el.set("default", self.element.attrib["default"]) required_codes = { - "0" : ("0", "1"), - "1" : ("1", "1"), - "+" : ("1", "unbounded"), - "*" : ("0", "unbounded"), - "-1" : ("0", "0") + "0": ("0", "1"), + "1": ("1", "1"), + "+": ("1", "unbounded"), + "*": ("0", "unbounded"), + "-1": ("0", "0"), } min_occurs, max_occurs = required_codes[self.element.attrib["required"]] el.set("minOccurs", min_occurs) @@ -198,7 +229,7 @@ def to_subtree(self): el = ElementTree.Element(_to_qname("xs:element")) el.set("name", other_root.attrib["name"]) - + el.set("type", f"{other_file.stem}:{name}Type") namespaces.append(f"{other_file.stem}") @@ -215,21 +246,20 @@ def to_subtree(self): if self.element.attrib["required"] == "1": el.set("type", f"{other_file.stem}:{name}Type") - #TODO: remove this line + # TODO: remove this line declared.setdefault("imports", set()).add(other_file.stem) - required_codes = { - "0" : ("0", "1"), - "1" : ("1", "1"), - "+" : ("1", "unbounded"), - "*" : ("0", "unbounded"), - "-1" : ("0", "0") + "0": ("0", "1"), + "1": ("1", "1"), + "+": ("1", "unbounded"), + "*": ("0", "unbounded"), + "-1": ("0", "0"), } min_occurs, max_occurs = required_codes[self.element.attrib["required"]] el.set("minOccurs", min_occurs) el.set("maxOccurs", max_occurs) - + return namespaces, el @@ -263,6 +293,7 @@ def to_subtree(self): return namespaces, el + @dataclass class ComplexType: element: ElementTree.Element @@ -301,7 +332,9 @@ def to_subtree(self): el.append(doc_el) if elements and "type" in self.element.attrib: - raise NotImplementedError("Cant handle sub-elements for an element declaring a type.") + raise NotImplementedError( + "Cant handle sub-elements for an element declaring a type." + ) elif "type" in self.element.attrib: el_type = _to_simple_type(self.element.attrib["type"]) extension = ElementTree.Element(_to_qname("xs:extension")) @@ -313,17 +346,35 @@ def to_subtree(self): simple_content.append(extension) el.append(simple_content) elif elements: - choice = ElementTree.Element(_to_qname("xs:choice")) - choice.set("maxOccurs", "unbounded") - choice.extend(elements) - el.append(choice) + has_unbound = any( + [el.attrib["maxOccurs"] in ["0", "unbounded"] for el in elements] + ) + if has_unbound: + # IMPORTANT NOTE: Using a choice container with + # maxOccurs="unbounded" is a last-resort option. The + # resulting XSD will ignore the min/maxOccurs of the contained + # elements and count any combination as valid. This means that + # the XSD becomes over-inclusive allowing invalid SDF to + # validate. + # + # I'm currently not sure how to avoid this, as it appears + # necessary given that some elements can occur an infinite + # number of times and they can do so in any order. + + container = ElementTree.Element(_to_qname("xs:choice")) + container.set("maxOccurs", "unbounded") + else: + container = ElementTree.Element(_to_qname("xs:all")) + container.extend(elements) + el.append(container) el.extend(attributes) else: el.extend(attributes) return namespaces, el -def setup_schema(used_ns:list, use_default_ns:bool=True) -> ElementTree.Element: + +def setup_schema(used_ns: list, use_default_ns: bool = True) -> ElementTree.Element: xsd_schema = ElementTree.Element(_to_qname("xs:schema")) if use_default_ns: @@ -339,18 +390,19 @@ def setup_schema(used_ns:list, use_default_ns:bool=True) -> ElementTree.Element: el.set("schemaLocation", f"./types.xsd") else: el.set("schemaLocation", f"./{name}Type.xsd") - + xsd_schema.append(el) xsd_schema.set(f"xmlns:{name}", namespaces[name]) return xsd_schema + out_dir = Path(args.target) if not out_dir.exists(): out_dir.mkdir(exist_ok=True, parents=True) # write type file -used_ns, element = ComplexType(root, root.attrib["name"]+"Type").to_subtree() +used_ns, element = ComplexType(root, root.attrib["name"] + "Type").to_subtree() xsd_schema = setup_schema(used_ns) xsd_schema.append(element) with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: From fa986c07164e4168012f146eba81c696b551ad0e Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 15:32:29 +0200 Subject: [PATCH 22/29] MAINT: remove dataclasses --- tools/xmlschema.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index ac9488851..85abdc4c7 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -102,7 +102,8 @@ def _to_qname(name: str) -> str: @dataclass class Description: - element: ElementTree.Element + def __init__(self, element: ElementTree.Element): + self.element = element def to_subtree(self): desc_text = self.element.text @@ -122,7 +123,8 @@ def to_subtree(self): @dataclass class Element: - element: ElementTree.Element + def __init__(self, element: ElementTree.Element): + self.element = element def to_basic(self): namespaces = list() @@ -219,7 +221,8 @@ def to_subtree(self): @dataclass class Include: - element: ElementTree.Element + def __init__(self, element: ElementTree.Element): + self.element = element def to_subtree(self): namespaces = list() @@ -265,7 +268,8 @@ def to_subtree(self): @dataclass class Attribute: - element: ElementTree.Element + def __init__(self, element: ElementTree.Element): + self.element = element def to_subtree(self): namespaces = list() @@ -294,10 +298,10 @@ def to_subtree(self): return namespaces, el -@dataclass class ComplexType: - element: ElementTree.Element - name: str = None + def __init__(self, element: ElementTree.Element, name: str = None): + self.element = element + self.name = name def to_subtree(self): namespaces = list() From d0b8836c8c6b4f262fd8f7851a77ac77ab842605 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 16:46:43 +0200 Subject: [PATCH 23/29] BUG: remove dataclass import --- tools/xmlschema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 85abdc4c7..7cb0f5685 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -2,7 +2,6 @@ from pathlib import Path import argparse -from dataclasses import dataclass from typing import List From bdb2ab7f22eca6e990cf5651b1ace256e13857be Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 16:48:40 +0200 Subject: [PATCH 24/29] MAINT: refactor complex type generation --- tools/xmlschema.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 7cb0f5685..d67d427ea 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -349,10 +349,18 @@ def to_subtree(self): simple_content.append(extension) el.append(simple_content) elif elements: - has_unbound = any( - [el.attrib["maxOccurs"] in ["0", "unbounded"] for el in elements] - ) - if has_unbound: + # drop depreciated elements + elements = [el for el in elements if not el.attrib["maxOccurs"] == "0"] + + unbounded_elements = [el for el in elements if el.attrib["maxOccurs"] == "unbounded"] + required_elements = [el for el in elements if el.attrib["minOccurs"] == "1"] + + if not unbounded_elements: + container = ElementTree.Element(_to_qname("xs:all")) + container.extend(elements) + el.append(container) + el.extend(attributes) + else: # IMPORTANT NOTE: Using a choice container with # maxOccurs="unbounded" is a last-resort option. The # resulting XSD will ignore the min/maxOccurs of the contained @@ -363,14 +371,15 @@ def to_subtree(self): # I'm currently not sure how to avoid this, as it appears # necessary given that some elements can occur an infinite # number of times and they can do so in any order. - container = ElementTree.Element(_to_qname("xs:choice")) container.set("maxOccurs", "unbounded") - else: - container = ElementTree.Element(_to_qname("xs:all")) - container.extend(elements) - el.append(container) - el.extend(attributes) + container.extend(elements) + el.append(container) + el.extend(attributes) + + # XSD 1.1 allows maxOccurs="unbound" within xs:all, so the + # code below works ... it won't work on XSD 1.0 though. + # container = ElementTree.Element(_to_qname("xs:all")) else: el.extend(attributes) @@ -379,6 +388,7 @@ def to_subtree(self): def setup_schema(used_ns: list, use_default_ns: bool = True) -> ElementTree.Element: xsd_schema = ElementTree.Element(_to_qname("xs:schema")) + xsd_schema.set("version", "1.1") if use_default_ns: xsd_schema.set("xmlns", namespaces[source.stem]) From e5a7042338f4c6fbc548a128c024e98635469ce9 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 17:12:25 +0200 Subject: [PATCH 25/29] BUG: remove dataclass decorators --- tools/xmlschema.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index d67d427ea..8cdf6a357 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -99,7 +99,6 @@ def _to_qname(name: str) -> str: return f"{{{namespaces[prefix]}}}{name}" -@dataclass class Description: def __init__(self, element: ElementTree.Element): self.element = element @@ -120,7 +119,6 @@ def to_subtree(self): return list(), annotation -@dataclass class Element: def __init__(self, element: ElementTree.Element): self.element = element @@ -218,7 +216,6 @@ def to_subtree(self): return namespaces, el -@dataclass class Include: def __init__(self, element: ElementTree.Element): self.element = element @@ -265,7 +262,6 @@ def to_subtree(self): return namespaces, el -@dataclass class Attribute: def __init__(self, element: ElementTree.Element): self.element = element From 4d6cf32204d596cc0a354afa25b2905db1c9ae6e Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Sun, 1 Aug 2021 17:53:45 +0200 Subject: [PATCH 26/29] MAINT: absolute pathing for namespace --- tools/xmlschema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 8cdf6a357..6d57debe8 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -35,7 +35,7 @@ cmd_arg_parser.add_argument( "--ns-prefix", dest="ns_prefix", - default="sdformat", + default="http://sdformat.org/schema", type=str, help="Prefix for generated xsd namespaces.", ) From b06c665ef5ca75f60e1e910f591fb5f93ef6f66a Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Mon, 2 Aug 2021 05:42:11 +0200 Subject: [PATCH 27/29] MAINT: handle non-ASCII inside sdf files --- tools/xmlschema.py | 85 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 6d57debe8..5aa5f9054 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -1,4 +1,6 @@ from xml.etree import ElementTree +from copy import deepcopy +from itertools import permutations, zip_longest from pathlib import Path import argparse @@ -347,35 +349,66 @@ def to_subtree(self): elif elements: # drop depreciated elements elements = [el for el in elements if not el.attrib["maxOccurs"] == "0"] - unbounded_elements = [el for el in elements if el.attrib["maxOccurs"] == "unbounded"] - required_elements = [el for el in elements if el.attrib["minOccurs"] == "1"] if not unbounded_elements: - container = ElementTree.Element(_to_qname("xs:all")) - container.extend(elements) - el.append(container) + seq_container = ElementTree.Element(_to_qname("xs:all")) + seq_container.extend(elements) + el.append(seq_container) el.extend(attributes) else: - # IMPORTANT NOTE: Using a choice container with - # maxOccurs="unbounded" is a last-resort option. The - # resulting XSD will ignore the min/maxOccurs of the contained - # elements and count any combination as valid. This means that - # the XSD becomes over-inclusive allowing invalid SDF to - # validate. - # - # I'm currently not sure how to avoid this, as it appears - # necessary given that some elements can occur an infinite - # number of times and they can do so in any order. - container = ElementTree.Element(_to_qname("xs:choice")) - container.set("maxOccurs", "unbounded") - container.extend(elements) - el.append(container) + # XSD 1.1 allows maxOccurs="unbound" within xs:all, so + # we could drop this entire else block if we could upgrade + # to XSD 1.1. (xmllint only supports XSD 1.0) + + # sort all elements into two categories: + # 1. maxOccurs <= 1 + # 2. maxOccurs = unbounded and minOccurs = 0 + # the case maxOccurs = unbounded and minOccurs = 1 is refactored + # into 1 required element in case 1 + 1 optional element in case 2 + # bounded_elements = [el for el in elements if not el.attrib["maxOccurs"] == "unbounded"] + # optional_unbounded = list() + # for unbounded_element in [el for el in elements if el.attrib["maxOccurs"] == "unbounded"]: + # unbounded_element:ElementTree.Element + # min_occurs = unbounded_element.attrib["minOccurs"] + # if min_occurs == "1": + # required_item = deepcopy(unbounded_element) + # required_item.set("maxOccurs", "1") + # bounded_elements.append(required_item) + + # # it will be ugly, but we can try to reduce clutter + # unbounded_element.attrib.pop("minOccurs") + # unbounded_element.attrib.pop("maxOccurs") + # optional_unbounded.append(unbounded_element) + + # infinity_choice = ElementTree.Element(_to_qname("xs:choice")) + # infinity_choice.set("maxOccurs", "unbounded") + # infinity_choice.extend(optional_unbounded) + + # # bounded elements may show up anywhere between unbounded optional elements. + # # However we can't use here (XSD 1.0 limitation). Instead use a choice + # # over sequences of unbounded choices with a permutation of bounded elements + # # inbetween them. + # container = ElementTree.Element(_to_qname("xs:choice")) + # for sequence in permutations(bounded_elements): + # seq_container = ElementTree.Element(_to_qname("xs:sequence")) + # seq_container.append(infinity_choice) + # for pair in zip_longest(sequence, [], fillvalue=infinity_choice): + # seq_container.extend(pair) + + # container.append(seq_container) + + # el.append(container) + # el.extend(attributes) + + # above code appears to work, but the generated + # XSD is multiple GB in size. Use this for now. + seq_container = ElementTree.Element(_to_qname("xs:choice")) + seq_container.set("maxOccurs", "unbounded") + seq_container.extend(elements) + el.append(seq_container) el.extend(attributes) - # XSD 1.1 allows maxOccurs="unbound" within xs:all, so the - # code below works ... it won't work on XSD 1.0 though. - # container = ElementTree.Element(_to_qname("xs:all")) else: el.extend(attributes) @@ -414,8 +447,8 @@ def setup_schema(used_ns: list, use_default_ns: bool = True) -> ElementTree.Elem used_ns, element = ComplexType(root, root.attrib["name"] + "Type").to_subtree() xsd_schema = setup_schema(used_ns) xsd_schema.append(element) -with open(out_dir / (source.stem + "Type.xsd"), "w") as out_file: - ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") +with open(out_dir / (source.stem + "Type.xsd"), "wb") as out_file: + ElementTree.ElementTree(xsd_schema).write(out_file, encoding="UTF-8", xml_declaration=True) # write element file file_name = source.stem @@ -425,5 +458,5 @@ def setup_schema(used_ns: list, use_default_ns: bool = True) -> ElementTree.Elem used_ns.append(file_name) xsd_schema = setup_schema(used_ns, use_default_ns=False) xsd_schema.append(element) -with open(out_dir / (source.stem + ".xsd"), "w") as out_file: - ElementTree.ElementTree(xsd_schema).write(out_file, encoding="unicode") +with open(out_dir / (source.stem + ".xsd"), "wb") as out_file: + ElementTree.ElementTree(xsd_schema).write(out_file, encoding="UTF-8", xml_declaration=True) From c471f41923021290c49550f50ec1a0f59ad3f3f9 Mon Sep 17 00:00:00 2001 From: FirefoxMetzger Date: Mon, 2 Aug 2021 06:34:54 +0200 Subject: [PATCH 28/29] TEST: add missing elements to double_pendulum.sdf --- test/sdf/double_pendulum.sdf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/sdf/double_pendulum.sdf b/test/sdf/double_pendulum.sdf index fd3d3b102..0a0b9052b 100644 --- a/test/sdf/double_pendulum.sdf +++ b/test/sdf/double_pendulum.sdf @@ -197,6 +197,11 @@ upper_link 1.0 0 0 + 0 + + 0 + 0.5 + @@ -205,6 +210,11 @@ lower_link 1.0 0 0 + 0 + + 0 + 0.5 + From cf1dad5e2098b7ffd0c9043f2207c53e726912ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Wallk=C3=B6tter?= Date: Thu, 31 Mar 2022 12:17:05 +0200 Subject: [PATCH 29/29] Apply suggestions from code review Co-authored-by: Bi0T1N --- tools/xmlschema.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/xmlschema.py b/tools/xmlschema.py index 5aa5f9054..cf53b79a4 100644 --- a/tools/xmlschema.py +++ b/tools/xmlschema.py @@ -14,6 +14,7 @@ "-i", "--in", dest="source", + metavar='FILE', required=True, type=str, help="SDF file inside of directory to compile.", @@ -32,7 +33,7 @@ dest="target", default=".", type=str, - help="Output directory for xsd file. Will be created if it doesn't exit.", + help="Output directory for XSD file. Will be created if it doesn't exist.", ) cmd_arg_parser.add_argument( "--ns-prefix", @@ -52,6 +53,13 @@ def _to_simple_type(in_type: str) -> str: +""" +Converts the input SDF type string to a XSD type string + +:param arg_type: input type +:raise: RuntimeError if no mapping exists +:returns: converted XSD string +""" known_types = { "unsigned int": "xs:unsignedInt", "unsigned long": "xs:unsignedLong",