diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4ca8d85..bb8be855 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,30 @@ jobs: echo ${CCACHE_BASEDIR} ccache -s fi - + + + - name: Set env vars for stubfiles + shell: bash + run: | + #Setup env + echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV + + + - name: Generate stubfiles + shell: bash + run: | + + #Install stubgen + ${{ steps.sofa.outputs.python_exe }} -m pip install mypy + + #For now use pybind11. This might be parametrized as an input of this action + ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d $WORKSPACE_INSTALL_PATH/lib/python3/site-packages -m Sofa --use_pybind11 + + #Go back to previous env + echo "PYTHONPATH=" | tee -a $GITHUB_ENV + + + - name: Set env vars for artifacts shell: bash run: | diff --git a/scripts/generate_stubs.py b/scripts/generate_stubs.py new file mode 100644 index 00000000..df41566f --- /dev/null +++ b/scripts/generate_stubs.py @@ -0,0 +1,35 @@ +import sys +import argparse +from utils import generate_module_stubs, generate_component_stubs + + +def main(site_package_dir,modules_name,use_pybind11 = False): + + work_dir = site_package_dir + modules = modules_name + + #Generate stubs using either pybind11-stubgen or mypy version of stubgen + + print(f"Generating stubgen for modules: {modules}") + + for module_name in modules: + generate_module_stubs(module_name, work_dir,use_pybind11) + + #Generate stubs for components using the factory + target_name="Sofa.Component" + generate_component_stubs(work_dir,target_name) + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog='generate_stubs', + description='Generates python stubs for SOFA modules') + + parser.add_argument('--use_pybind11',action='store_true',help='If flag is present, will use pybind11-stubgen instead of mypy stugen') + parser.add_argument('-d','--site_package_dir',nargs=1,help='Path to the site-package folder containing the SOFA modules') + parser.add_argument('-m','--modules_name',nargs='+',help='List of modules names to generate stubs for') + + args = parser.parse_args() + + main(args.site_package_dir,args.modules_name,args.use_pybind11) diff --git a/scripts/sofaStubgen.py b/scripts/sofaStubgen.py new file mode 100644 index 00000000..74891766 --- /dev/null +++ b/scripts/sofaStubgen.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -*- +""" +Autogenerate python wrapper for SOFA objects. + +Contributors: + damien.marchal@univ-lille.fr +""" +import re +import sys, os +import pprint +import argparse + +import Sofa +import Sofa.Simulation + +# some data field cannot have names as this is a python keywoards. +reserved = ["in", "with", "for", "if", "def", "class", "global"] + +def sofa_to_python_typename(name, short=False): + t = {"string":"str", + "bool": "bool", + "TagSet" : "object", + "BoundingBox" : "object", + "ComponentState" : "object", + "RGBAColor" : "object", + "OptionsGroup" : "object", + "SelectableItem" : "object", + "Material" : "object", + "DisplayFlags" : "object", + "d" : "float", + "f" : "float", + "i" : "int", + "I" : "int", + "L" : "int", + "l" : "int", + "b" : "int", + "LinkPath" : "LinkPath" + } + + SofaArray = "TypeHints.SofaArray" + if short: + SofaArray = "TypeHints.SofaArray" + + if name in t: + return t[name] + elif "Rigid" in name: + return SofaArray + elif "vector" in name: + return SofaArray + elif "set" in name: + return SofaArray + elif "Vec" in name: + return SofaArray + elif "Mat" in name: + return SofaArray + elif "Quat" in name: + return SofaArray + elif "map" in name: + return "object" + elif "fixed_array" in name: + return SofaArray + raise Exception("Missing type name,... ", name) + + +def sofa_datafields_to_constructor_arguments_list(data_fields, object_name, has_template=True): + #{'defaultValue': '0', + # 'group': '', + # 'help': 'if true, handle the events, otherwise ignore ' + # 'the events', + # 'name': 'listening', + # 'type': 'bool'}, + required_data_fields = [] + optional_data_fields = [] + + for data_field in data_fields: + name = data_field["name"] + if name in reserved: + print(f"Warning: {object_name} contains a the data field named '{name}' which is also a python keyword") + continue + + if " " in name: + print(f"Warning: this is an invalid arguments name: '{name}'") + continue + + if len(name) == 0: + print("Warning: empty data field empty name") + continue + + if data_field["isRequired"]: + required_data_fields.append(data_field) + else: + optional_data_fields.append(data_field) + + result_params = "" + if has_template: + result_params = "template: Optional[str] = None, " + result_params += ",".join([data["name"]+": "+sofa_to_python_typename(data["type"]) for data in required_data_fields]) + result_params += ",".join([data["name"]+": Optional["+sofa_to_python_typename(data["type"])+"] = None" for data in optional_data_fields]) + + ordered_fields = sorted(data_fields, key=lambda x: x["name"]) + + all = "\n ".join( [ data["name"]+" ("+sofa_to_python_typename(data["type"], short=True)+"): " + data["help"] for data in ordered_fields]) + all += "\n template (str): the type of degree of freedom" + return result_params, all + +def sofa_datafields_to_doc(data_fields): + p = "" + for data in data_fields: + name = data["name"] + help = data["help"] + if len(name) == 0: + continue + if " " in name: + continue + + if name in reserved: + p += "\t\t " + name + ": " + help + " (NB: use the kwargs syntax as name is a reserved word in python)\n\n" + else: + p += "\t\t " + name + ": " + help + "\n\n" + + return p + +def clean_sofa_text(t): + t = t.replace("\n","") + t = t.replace("'", "") + return t + +def sofa_datafields_to_typehints(data_fields, mode="Sofa"): + p = "" + p2 = "" + for data in data_fields: + name = data["name"] + help = clean_sofa_text(data["help"]) + type = sofa_to_python_typename(data["type"]) + if len(name) == 0: + continue + + if " " in name: + continue + + if name in reserved: + p += " " + name + ": " + help + " (NB: use the kwargs syntax as name is a reserved word in python)\n\n" + else: + p += f" {name}: Data[{type}] \n '{help}'\n\n" + p2 += f" {name}: Optional[{type} | LinkPath] = None \n '{help}'\n\n" + + return p, p2 + +def make_all_init_files(root_dir): + entries = {} + for dirpath, dirnames, filenames in os.walk(root_dir): + # Test if the root_dir and the dirpath are the same to skip first entry + # This could be implemented by using [os.walk(root_dir)][1:] but in that case + # python type hints get lost on my version 3.10. Maybe in a future python release + # this case will be handled + if os.path.abspath(root_dir) == os.path.abspath(dirpath): + continue + + initfile = open(dirpath + "/__init__.py", "wt") + res = [] + fres = [] + fres2 = [] + for d in dirnames: + res.append(d) + for f in filenames: + if f.endswith(".py") and f != "__init__.py": + fres.append(f[:-3]) + res.append(f[:-3]) + listc = "" + for r in res: + listc += " " + str(r) + "\n" + + initfile.write("# -*- coding: utf-8 -*-\n\n") + initfile.write(""" +\"\"\" +Sofa Component %s + +Summary: +======== + +%s + +\"\"\" +""" % (os.path.basename(dirpath), listc)) + +def wrapper_code(class_name, description, data_list, properties_doc, + class_typehints, params_typehints, + constructor_params_typehints, + constructor_params_docs): + properties_doc = "" + return f""" +class {class_name}(Object): + \"\"\"{description}\"\"\" + + def __init__(self, {constructor_params_typehints}): + \"\"\"{description} + + Args: + {constructor_params_docs} + \"\"\" + ... + +{class_typehints} + + @dataclasses.dataclass + class Parameters: + \"\"\"Parameter for the construction of the {class_name} component\"\"\" + +{params_typehints} + + def to_dict(self): + return dataclasses.asdict(self) + + @staticmethod + def new_parameters() -> Parameters: + return {class_name}.Parameters() +""" + +def documentation_code(class_name): + return """ +\"\"\" +Component %s + +.. autofunction:: %s + +Indices and tables +****************** + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` +\"\"\" +import Sofa +from Sofa.Core import Object, LinkPath +from Sofa.Component.TypeHints import Data, Optional, SofaArray +import dataclasses +""" % (class_name, class_name) + +def select_single_template(template_list): + """ returns a single template""" + for dim in ["Vec3", "Rigid3", "Vec2", "Rigid2", "Vec1", "Quatd"]: + for template in template_list: + if "Cuda" not in template and dim in template: + return template + + for template in template_list: + return template + + return "" + + +def create_typhint(pathname): + from pathlib import Path + path = Path(pathname).parent + if not os.path.exists(str(path)): + path.mkdir(parents=True) + c=""" +from typing import TypeVar, Generic, Optional as __optional__ +import numpy.typing +from Sofa.Core import Node, Object, LinkPath +import numpy +from numpy.typing import ArrayLike + +T = TypeVar("T", bound=object) + +# This is a generic type 'T' implemented without PEP 695 (as it needs python 3.12) +class Data(Generic[T]): + linkpath: LinkPath + value: T + +Optional = __optional__ +SofaArray = numpy.ndarray | list +""" + with open(pathname,"w") as w: + w.write(c) + +from pprint import pprint + +def load_component_list(target_name): + import json + import Sofa + import Sofa.Core + import SofaRuntime + + if target_name in ["Sofa"]: + # The binding is not a sofa plugin. + print("Loading a python module") + else: + print("Loading a sofa plugin") + SofaRuntime.importPlugin(target_name) + + json = json.loads(Sofa.Core.ObjectFactory.dump_json()) + + selected_entries = [] + for item in json: + selected_item = None + for type, entry in item["creator"].items(): + if entry["target"].startswith(target_name): + for data in entry["object"]["data"]: + data["isRequired"] = True + selected_item = item + + if selected_item: + selected_entries.append(selected_item) + + print("Number of objects ", len(json)) + print("Number of selected objects ", len(selected_entries)) + + return selected_entries + +def create_sofa_stubs(code_model, target_path): + blacklist = ["RequiredPlugin"] + + template_nodes = dict() + not_created_objects = list() + + create_typhint(target_path + "Sofa/Component/TypeHints.py") + + for entry in code_model: + data = {} + + class_name = entry["className"] + entry_templates = [n for n in entry["creator"].keys()] + description = entry["description"] + + selected_template = select_single_template(entry_templates) + selected_entry = entry["creator"][selected_template] + selected_class = selected_entry["class"] + selected_object = selected_entry["object"] + selected_target = selected_entry["target"] + + if len(selected_target) == 0: + selected_target = "unknown_target" + + object_name = class_name + "<" + selected_template + ">" + if object_name in blacklist or class_name in blacklist: + print("Skipping ", object_name) + continue + + + data = selected_object["data"] # obj.getDataFields() + links = selected_object["link"] + target = selected_target + + #load_existing_stub(selected_target, class_name) + + arguments_list, constructor_params_doc = sofa_datafields_to_constructor_arguments_list(data, object_name, len(entry_templates) > 0) + params_doc = sofa_datafields_to_doc(data) + class_typehint, params_typehint = sofa_datafields_to_typehints(data) + code = wrapper_code(class_name, description.strip(), arguments_list, params_doc, + class_typehint, params_typehint, + arguments_list, constructor_params_doc) + + pathname = target_path + target.replace(".","/") + "/" + full_component_name = target + "." + class_name + + # Creates the destination directory if needed. + os.makedirs(pathname, exist_ok=True) + + outfile = open(pathname + class_name + ".py", "wt") + outfile.write("# -*- coding: utf-8 -*-\n\n") + outfile.write(documentation_code(class_name)) + outfile.write(code) + outfile.close() + + if full_component_name in code_model: + raise Exception("Already existing entry") + + # In every directory, scan the object that are in and generates an init.py file + make_all_init_files(target_path) + return code_model + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog='sofa-component-stub-generator', + description='Generates python stubs that describes sofa components that have no binding') + parser.add_argument('--target_name', default="Sofa.Component") + parser.add_argument('--output_directory', default="out/") + args = parser.parse_args() + + target_name = args.target_name + output_directory = args.output_directory + + print(f"Generating SOFA's components python interfaces for {target_name}") + + components = load_component_list(target_name) + create_sofa_stubs(components, output_directory) + diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 00000000..9e24356d --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,96 @@ +import sys +import os +import shutil +from sofaStubgen import load_component_list, create_sofa_stubs +from mypy.stubgen import parse_options, generate_stubs +import argparse + + + +#Method to use pybind11-stubgen +def pybind11_stub(module_name: str): + import logging + from pybind11_stubgen import CLIArgs, stub_parser_from_args, Printer, to_output_and_subdir, run, Writer + logging.basicConfig( + level=logging.INFO, + format="%(name)s - [%(levelname)7s] %(message)s", + ) + args = CLIArgs( + module_name=module_name, + output_dir='./out', + stub_extension="pyi", + # default ags: + root_suffix=None, + ignore_all_errors=False, + ignore_invalid_identifiers=None, + ignore_invalid_expressions=None, + ignore_unresolved_names=None, + exit_code=False, + numpy_array_wrap_with_annotated=False, + numpy_array_use_type_var=False, + numpy_array_remove_parameters=False, + enum_class_locations=[], + print_safe_value_reprs=None, + print_invalid_expressions_as_is=False, + dry_run=False) + + parser = stub_parser_from_args(args) + printer = Printer(invalid_expr_as_ellipses=not args.print_invalid_expressions_as_is) + + out_dir, sub_dir = to_output_and_subdir( + output_dir=args.output_dir, + module_name=args.module_name, + root_suffix=args.root_suffix, + ) + + run( + parser, + printer, + args.module_name, + out_dir, + sub_dir=sub_dir, + dry_run=args.dry_run, + writer=Writer(stub_ext=args.stub_extension), + ) + +#Generate stubs using either pybind11-stubgen or mypy version of stubgen +def generate_module_stubs(module_name, work_dir, usePybind11_stubgen = False): + print(f"Generating stubgen for {module_name} in {work_dir}") + + if(usePybind11_stubgen): + #Use pybind11 stubgen + #Could be replaced by an os call to + #subprocess.run(["pybind11-stubgen", module_name, "-o", "out"], check=True) + pybind11_stub(module_name) + else: + #Use mypy stubgen + options = parse_options(["-v","-p",module_name,"--include-docstrings","--no-analysis", "--ignore-errors"]) + generate_stubs(options) + + module_out_dir = os.path.join("out", module_name) + target_dir = os.path.join(work_dir, module_name) + + + if os.path.isdir(module_out_dir): + shutil.copytree(module_out_dir, target_dir, dirs_exist_ok=True) + print(f"Resync terminated for copying '{module_name}' to '{target_dir}'") + + shutil.rmtree("out", ignore_errors=True) + +#Generate stubs for components using the factory +def generate_component_stubs(work_dir,target_name): + print(f"Generating stubgen for all components in Sofa.Components using custom sofaStubgen.py") + + components = load_component_list(target_name) + create_sofa_stubs(components, "out/") + + + sofa_out_dir = os.path.join("out", "Sofa") + target_dir = os.path.join(work_dir, "Sofa") + + + if os.path.isdir(sofa_out_dir): + shutil.copytree(sofa_out_dir, target_dir, dirs_exist_ok=True) + print("Resync terminated.") + + shutil.rmtree("out", ignore_errors=True)