diff --git a/setup.py b/setup.py index f568894..32d6b09 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,11 @@ with open("README.rst", "r") as f_readme: long_desc = f_readme.read() -requires = ["Sphinx>=4.0.0", "Pygments>=2.0.1"] +requires = [ + "Sphinx>=4.0.0", + "tree-sitter-matlab>=1.0.2", + "tree-sitter>=0.21.3,<0.23.0", +] setup( name="sphinxcontrib-matlabdomain", diff --git a/sphinxcontrib/mat_documenters.py b/sphinxcontrib/mat_documenters.py index b5cd00c..4d5e51a 100644 --- a/sphinxcontrib/mat_documenters.py +++ b/sphinxcontrib/mat_documenters.py @@ -14,6 +14,7 @@ MatFunction, MatClass, MatProperty, + MatEnumeration, MatMethod, MatScript, MatException, @@ -555,6 +556,9 @@ def member_is_friend_of(member, friends): else: return False + def member_is_enum(member): + return isinstance(member, MatEnumeration) + ret = [] # search for members in source code too @@ -637,7 +641,7 @@ def member_is_friend_of(member, friends): isattr = True else: # ignore undocumented members if :undoc-members: is not given - keep = has_doc or self.options.undoc_members + keep = has_doc or self.options.undoc_members or member_is_enum(member) # give the user a chance to decide whether this member # should be skipped @@ -656,7 +660,6 @@ def member_is_friend_of(member, friends): if keep: ret.append((membername, member, isattr)) - return ret def document_members(self, all_members=False): @@ -1229,11 +1232,19 @@ def document_members(self, all_members=False): for (membername, member) in filtered_members if isinstance(member, MatMethod) and member.name != member.cls.name ] + # create list of enums + enum_names = [ + membername + for (membername, member) in filtered_members + if isinstance(member, MatEnumeration) + ] # create list of other members other_names = [ membername for (membername, member) in filtered_members - if not isinstance(member, MatMethod) and not isinstance(member, MatProperty) + if not isinstance(member, MatMethod) + and not isinstance(member, MatProperty) + and not isinstance(member, MatEnumeration) # exclude parent modules with names matching members (as in Myclass.Myclass) and not (hasattr(member, "module") and member.name == member.module) ] @@ -1255,6 +1266,12 @@ def document_members(self, all_members=False): for (membername, member) in members if not isinstance(member, MatMethod) or member.name == member.cls.name ] + # create list of members that are not properties + non_enums = [ + membername + for (membername, member) in members + if not isinstance(member, MatEnumeration) + ] # create list of members that are not non-constructor methods non_other = [ membername @@ -1281,6 +1298,10 @@ def document_members(self, all_members=False): "Property Summary", non_properties, all_members ) + # enumss + if enum_names: + self.document_member_section("Enumeration Values", non_enums, all_members) + # methods if meth_names: self.document_member_section("Method Summary", non_methods, all_members) @@ -1359,10 +1380,11 @@ def format_args(self): is_ctor = self.object.cls.name == self.object.name if self.object.args: - if self.object.args[0] in ("obj", "self") and not is_ctor: - return "(" + ", ".join(self.object.args[1:]) + ")" + arglist = list(self.object.args.keys()) + if arglist[0] in ("obj", "self") and not is_ctor: + return "(" + ", ".join(arglist[1:]) + ")" else: - return "(" + ", ".join(self.object.args) + ")" + return "(" + ", ".join(arglist) + ")" def document_members(self, all_members=False): pass @@ -1453,7 +1475,16 @@ def add_directive_header(self, sig): obj_default = " = " + obj_default if self.env.config.matlab_show_property_specs: - obj_default = self.object.specs + obj_default + prop_spec = "" + if self.object.size is not None: + prop_spec = prop_spec + "(" + ",".join(self.object.size) + ")" + if self.object.type is not None: + prop_spec = prop_spec + " " + self.object.type + if self.object.validators is not None: + prop_spec = ( + prop_spec + " {" + ",".join(self.object.validators) + "}" + ) + obj_default = prop_spec + obj_default self.add_line(" :annotation: " + obj_default, "") elif self.options.annotation is SUPPRESS: diff --git a/sphinxcontrib/mat_parser.py b/sphinxcontrib/mat_parser.py deleted file mode 100644 index 55db502..0000000 --- a/sphinxcontrib/mat_parser.py +++ /dev/null @@ -1,88 +0,0 @@ -""" - sphinxcontrib.mat_parser - ~~~~~~~~~~~~~~~~~~~~~~~~ - - Functions for parsing MatlabLexer output. - - :copyright: Copyright 2023-2024 by the sphinxcontrib-matlabdomain team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import re -import sphinx.util - -logger = sphinx.util.logging.getLogger("matlab-domain") - - -def remove_comment_header(code): - """ - Removes the comment header (if there is one) and empty lines from the - top of the current read code. - :param code: Current code string. - :type code: str - :returns: Code string without comments above a function, class or - procedure/script. - """ - # get the line number when the comment header ends (incl. empty lines) - ln_pos = 0 - for line in code.splitlines(True): - if re.match(r"[ \t]*(%|\n)", line): - ln_pos += 1 - else: - break - - if ln_pos > 0: - # remove the header block and empty lines from the top of the code - try: - code = code.split("\n", ln_pos)[ln_pos:][0] - except IndexError: - # only header and empty lines. - code = "" - - return code - - -def remove_line_continuations(code): - """ - Removes line continuations (...) from code as functions must be on a - single line - :param code: - :type code: str - :return: - """ - # pat = r"('.*)(\.\.\.)(.*')" - # code = re.sub(pat, r"\g<1>\g<3>", code, flags=re.MULTILINE) - - pat = r"^([^%'\"\n]*)(\.\.\..*\n)" - code = re.sub(pat, r"\g<1>", code, flags=re.MULTILINE) - return code - - -def fix_function_signatures(code): - """ - Transforms function signatures with line continuations to a function - on a single line with () appended. Required because pygments cannot - handle this situation correctly. - - :param code: - :type code: str - :return: Code string with functions on single line - """ - pat = r"""^[ \t]*function[ \t.\n]* # keyword (function) - (\[?[\w, \t.\n]*\]?) # outputs: group(1) - [ \t.\n]*=[ \t.\n]* # punctuation (eq) - (\w+)[ \t.\n]* # name: group(2) - \(?([\w, \t.\n]*)\)?""" # args: group(3) - pat = re.compile(pat, re.X | re.MULTILINE) # search start of every line - - # replacement function - def repl(m): - retv = m.group(0) - # if no args and doesn't end with parentheses, append "()" - if not (m.group(3) or m.group(0).endswith("()")): - retv = retv.replace(m.group(2), m.group(2) + "()") - return retv - - code = pat.sub(repl, code) # search for functions and apply replacement - - return code diff --git a/sphinxcontrib/mat_tree_sitter_parser.py b/sphinxcontrib/mat_tree_sitter_parser.py new file mode 100644 index 0000000..4460691 --- /dev/null +++ b/sphinxcontrib/mat_tree_sitter_parser.py @@ -0,0 +1,929 @@ +from importlib.metadata import version +import tree_sitter_matlab as tsml +from tree_sitter import Language, Parser +import re + +# Attribute default dictionary used to give default values for e.g. `Abstract` or `Static` when used without +# a right hand side i.e. `classdef (Abstract)` vs `classdef (Abstract=true)` +# From: +# - http://www.mathworks.com/help/matlab/matlab_oop/class-attributes.html +# - https://mathworks.com/help/matlab/matlab_oop/property-attributes.html +# - https://mathworks.com/help/matlab/matlab_prog/define-property-attributes-1.htm +# - https://mathworks.com/help/matlab/matlab_oop/method-attributes.html +# - https://mathworks.com/help/matlab/ref/matlab.unittest.testcase-class.html +MATLAB_ATTRIBUTE_DEFAULTS = { + "AbortSet": True, + "Abstract": True, + "ClassSetupParameter": True, + "Constant": True, + "ConstructOnLoad": True, + "Dependent": True, + "DiscreteState": True, + "GetObservable": True, + "HandleCompatible": True, + "Hidden": True, + "MethodSetupParameter": True, + "NonCopyable": True, + "Nontunable": True, + "PartialMatchPriority": True, + "Sealed": True, + "SetObservable": True, + "Static": True, + "Test": None, + "TestClassSetup": None, + "TestClassTeardown": None, + "TestMethodSetup": None, + "TestMethodTeardown": None, + "TestParameter": None, + "Transient": True, +} + + +tree_sitter_ver = tuple([int(sec) for sec in version("tree_sitter").split(".")]) +if tree_sitter_ver[1] == 21: + ML_LANG = Language(tsml.language(), "matlab") +else: + ML_LANG = Language(tsml.language()) + +# QUERIES +q_classdef = ML_LANG.query( + """(class_definition + . + "classdef" + . + (attributes + [(attribute) @attrs _]+ + )? + . + (identifier) @name + . + (superclasses + [(property_name) @supers _]+ + )? + . + (comment)? @docstring + ) @class +""" +) + +q_attributes = ML_LANG.query( + """(attribute + (identifier) @name + [ + (identifier) @value + (string) @value + (metaclass_operator) @value + (cell (row [(metaclass_operator) @value _]*)) + (cell (row [(string) @value _]*)) + ]? @rhs + ) + """ +) + +q_supers = ML_LANG.query("""[(identifier) @secs "."]+ """) + +q_properties = ML_LANG.query( + """(properties + . + (attributes + [(attribute) @attrs _]+ + )? + [(property) @properties _]+ + ) @prop_block +""" +) + +q_methods = ML_LANG.query( + """(methods + (attributes + [(attribute) @attrs _]+ + )? + [(function_definition) @methods _]+ + ) @meth_block +""" +) + +q_enumerations = ML_LANG.query( + """(enumeration + [(enum) @enums _]+ + ) @enum_block +""" +) + +q_events = ML_LANG.query( + """(events + (attributes + [(attribute) @attrs _]+ + )? + (identifier)+ @events + ) @event_block +""" +) + +q_property = ML_LANG.query( + """ + (property name: (identifier) @name + (dimensions + [(spread_operator) @dims (number) @dims _]+ + )? + (identifier)? @type + . + (identifier)? @size_type + (validation_functions + [(identifier) @validation_functions (function_call) @validation_functions _]+ + )? + (default_value)? @default + (comment)? @docstring + ) +""" +) + +q_old_property = ML_LANG.query( + """ + (property name: (identifier) @name + (identifier) @type + (identifier)? @size_type + (default_value)? @default + (comment)? @docstring + ) +""" +) + +q_enum = ML_LANG.query( + """(enum + . + (identifier) @name + [(_) @args _]* + ) +""" +) + +q_fun = ML_LANG.query( + """(function_definition + _* + (function_output + [ + (identifier) @outputs + (multioutput_variable + [[(identifier) (ignored_argument)] @outputs _]+ + ) + ] + )? + _* + name: (identifier) @name + _* + (function_arguments + [(identifier) @params (ignored_argument) @params _]* + )? + _* + [(arguments_statement) @argblocks _]* + . + (comment)? @docstring + ) +""" +) + +q_argblock = ML_LANG.query( + """ + (arguments_statement + . + (attributes + [(identifier) @attrs _]* + )? + . + [(property) @args _]* + ) +""" +) + +q_arg = ML_LANG.query( + """ + (property name: + [ + (identifier) @name + (property_name + [(identifier) @name _]+ + ) + ] + (dimensions + [(spread_operator) @dims (number) @dims _]+ + )? + (identifier)? @type + (validation_functions + [[(identifier) (function_call)] @validation_functions _]+ + )? + (default_value [(number) (identifier)])? @default + (comment)? @docstring + ) +""" +) + +q_script = ML_LANG.query( + """ + (source_file + (comment)? @docstring + ) + """ +) + +q_get_set = ML_LANG.query("""["get." "set."]""") + +q_line_continuation = ML_LANG.query("(line_continuation) @lc") + + +re_percent_remove = re.compile(r"^[ \t]*% ?", flags=re.M) +re_trim_line = re.compile(r"^[ \t]*", flags=re.M) +re_assign_remove = re.compile(r"^=[ \t]*") + + +def tree_sitter_is_0_21(): + """Check if tree-sitter is v0.21.* in order to use the correct language initialization and syntax.""" + if not hasattr(tree_sitter_is_0_21, "is_21"): + tree_sitter_ver = tuple([int(sec) for sec in version("tree_sitter").split(".")]) + tree_sitter_is_0_21.is_21 = tree_sitter_ver[1] == 21 # memoize + return tree_sitter_is_0_21.is_21 + + +def get_row(point): + """Get row from point. This api changed from v0.21.3 to v0.22.0""" + if tree_sitter_is_0_21(): + return point[0] + else: + return point.row + + +def process_text_into_docstring(text, encoding): + """Take a text bytestring and decode it into a docstring.""" + docstring = text.decode(encoding, errors="backslashreplace") + return re.sub(re_percent_remove, "", docstring) + + +def process_default(node, encoding): + """Take the node defining a default and remove any line continuations before generating the default.""" + text = node.text + to_keep = set(range(node.end_byte - node.start_byte)) + lc_matches = q_line_continuation.matches(node) + for _, match in lc_matches: + # TODO this copies a lot perhaps there is a better option. + lc = match["lc"] + cut_start = lc.start_byte - node.start_byte + cut_end = lc.end_byte - node.start_byte + to_keep -= set(range(cut_start, cut_end)) + # NOTE: hardcoded endianess is fine because for one byte this does not matter. + # See python bikeshed on possible defaults for this here: + # https://discuss.python.org/t/what-should-be-the-default-value-for-int-to-bytes-byteorder/10616 + new_text = b"".join( + [byte.to_bytes(1, "big") for idx, byte in enumerate(text) if idx in to_keep] + ) + # TODO We may want to do an in-order traversal of the parse here to generate a "nice" reformatted single line + # however doing so sufficiently generically is likely a major undertaking. + default = new_text.decode(encoding, errors="backslashreplace") + default = re.sub(re_assign_remove, "", default) + return re.sub(re_trim_line, "", default) + + +class MatScriptParser: + def __init__(self, root_node, encoding): + """Parse m script""" + self.encoding = encoding + script_matches = q_script.matches(root_node) + if script_matches: + _, script_match = q_script.matches(root_node)[0] + docstring_node = script_match.get("docstring") + if docstring_node is not None: + self.docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + else: + self.docstring = None + else: + self.docstring = None + + +class MatFunctionParser: + def __init__(self, root_node, encoding): + """Parse Function definition""" + self.encoding = encoding + _, fun_match = q_fun.matches(root_node)[0] + self.name = fun_match.get("name").text.decode( + self.encoding, errors="backslashreplace" + ) + + # Get outputs (possibly more than one) + self.retv = {} + output_nodes = fun_match.get("outputs") + if output_nodes is not None: + retv = [ + output.text.decode(self.encoding, errors="backslashreplace") + for output in output_nodes + ] + for output in retv: + self.retv[output] = {} + + # Get parameters + self.args = {} + arg_nodes = fun_match.get("params") + if arg_nodes is not None: + args = [ + arg.text.decode(self.encoding, errors="backslashreplace") + for arg in arg_nodes + ] + for arg in args: + self.args[arg] = {} + + # parse out info from argument blocks + argblock_nodes = fun_match.get("argblocks") + if argblock_nodes is not None: + for argblock_node in argblock_nodes: + self._parse_argument_section(argblock_node) + + # get docstring + docstring_node = fun_match.get("docstring") + docstring = "" + if docstring_node is not None: + prev_sib = docstring_node.prev_named_sibling + if get_row(docstring_node.start_point) - get_row(prev_sib.end_point) <= 1: + if get_row(docstring_node.start_point) == get_row(prev_sib.end_point): + # if the docstring is on the same line as the end of the function drop it + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + split_ds = docstring.split("\n") + docstring = "\n".join(split_ds[1:]) if len(split_ds) > 1 else "" + else: + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + + if not docstring: + docstring = None + self.docstring = docstring + + def _parse_argument_section(self, argblock_node): + _, argblock_match = q_argblock.matches(argblock_node)[0] + attrs_nodes = argblock_match.get("attrs") + attrs = self._parse_attributes(attrs_nodes) + + arguments = argblock_match.get("args") + + # TODO this is almost identical to property parsing. + # might be a good idea to extract common code here. + for arg in arguments: + # match property to extract details + _, arg_match = q_arg.matches(arg)[0] + + # extract name (this is always available so no need for None check) + name = [ + name.text.decode(self.encoding, errors="backslashreplace") + for name in arg_match.get("name") + ] + + # extract dims list + dims_list = arg_match.get("dims") + dims = None + if dims_list is not None: + dims = tuple( + [ + dim.text.decode(self.encoding, errors="backslashreplace") + for dim in dims_list + ] + ) + + # extract type + type_node = arg_match.get("type") + typename = ( + type_node.text.decode(self.encoding, errors="backslashreplace") + if type_node is not None + else None + ) + + # extract validator functions + vf_list = arg_match.get("validation_functions") + vfs = None + if vf_list is not None: + vfs = [ + vf.text.decode(self.encoding, errors="backslashreplace") + for vf in vf_list + ] + + # extract default + default_node = arg_match.get("default") + default = ( + process_default(default_node, self.encoding) + if default_node is not None + else None + ) + + # extract inline or following docstring if there is no semicolon + docstring_node = arg_match.get("docstring") + docstring = "" + if docstring_node is not None: + # tree-sitter-matlab combines inline comments with following + # comments which means this requires some relatively ugly + # processing, but worth it for the ease of the rest of it. + prev_sib = docstring_node.prev_named_sibling + if get_row(docstring_node.start_point) == get_row(prev_sib.end_point): + # if the docstring is on the same line as the end of the definition only take the inline part + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + docstring = docstring.split("\n")[0] + elif ( + get_row(docstring_node.start_point) - get_row(prev_sib.end_point) + <= 1 + ): + # Otherwise take the whole docstring + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + + # extract inline or following docstring if there _is_ a semicolon. + # this is only done if we didn't already find a docstring with the previous approach + next_node = arg.next_named_sibling + if next_node is None or docstring is not None: + # Nothing to be done. + pass + elif next_node.type == "comment": + if get_row(next_node.start_point) == get_row(arg.end_point): + # if the docstring is on the same line as the end of the definition only take the inline part + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + docstring = docstring.split("\n")[0] + elif get_row(next_node.start_point) - get_row(arg.end_point) <= 1: + # Otherwise take the whole docstring + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + + # override docstring with prior if exists + prev_node = arg.prev_named_sibling + if prev_node is None: + # Nothing we can do, no previous comment + pass + elif prev_node.type == "comment": + # We have a previous comment if it ends on the previous + # line then we set the docstring. We also need to check + # if the first line of the comment is the same as a + # previous argument. + if get_row(arg.start_point) - get_row(prev_node.end_point) <= 1: + ds = process_text_into_docstring(prev_node.text, self.encoding) + prev_arg = prev_node.prev_named_sibling + if prev_arg is not None and prev_arg.type == "property": + if get_row(prev_node.start_point) == get_row( + prev_arg.end_point + ): + ds = "\n".join(ds.split("\n")[1:]) + if ds: + docstring = ds + else: + if get_row(arg.start_point) - get_row(prev_node.end_point) <= 1: + docstring = process_text_into_docstring( + prev_node.text, self.encoding + ) + elif prev_node.type == "property": + # The previous argumentnode may have eaten our comment + # check for it a trailing comment. If it is not there + # then we stop looking. + prev_comment = prev_node.named_children[-1] + if prev_comment.type == "comment": + # we now need to check if prev_comment ends on the line + # before ours and trim the first line if it on the same + # line as prev property. + if get_row(arg.start_point) - get_row(prev_comment.end_point) <= 1: + ds = process_text_into_docstring( + prev_comment.text, self.encoding + ) + if get_row(prev_comment.start_point) == get_row( + prev_comment.prev_named_sibling.end_point + ): + ds = "\n".join(ds.split("\n")[1:]) + if ds: + docstring = ds + # After all that if our docstring is empty then we have none + if not docstring.strip(): + docstring = None + else: + pass # docstring = docstring.rstrip() + + # Here we trust that the person is giving us valid matlab. + if "Output" in attrs.keys(): + arg_loc = self.retv + else: + arg_loc = self.args + if len(name) == 1: + arg_loc[name[0]] = { + "attrs": attrs, + "size": dims, + "type": typename, + "validators": vfs, + "default": default, + "docstring": docstring, + } + else: + # how to handle dotted args + pass + + def _parse_attributes(self, attrs_nodes): + attrs = {} + if attrs_nodes is not None: + for attr_node in attrs_nodes: + name = attr_node.text.decode(self.encoding, errors="backslashreplace") + attrs[name] = None + return attrs + + +class MatClassParser: + def __init__(self, root_node, encoding): + # DATA + self.encoding = encoding + self.name = "" + self.supers = [] + self.attrs = {} + self.docstring = "" + self.properties = {} + self.methods = {} + self.enumerations = {} + self.events = {} + + self.root_node = root_node + + # Parse class basics + class_matches = q_classdef.matches(root_node) + _, class_match = class_matches[0] + self.cls = class_match.get("class") + self.name = class_match.get("name") + + # Parse class attrs and supers + attrs_nodes = class_match.get("attrs") + self.attrs = self._parse_attributes(attrs_nodes) + + supers_nodes = class_match.get("supers") + if supers_nodes is not None: + for super_node in supers_nodes: + _, super_match = q_supers.matches(super_node)[0] + super_cls = [ + sec.text.decode(self.encoding, errors="backslashreplace") + for sec in super_match.get("secs") + ] + self.supers.append(".".join(super_cls)) + + # get docstring and check that it consecutive + docstring_node = class_match.get("docstring") + docstring = "" + if docstring_node is not None: + prev_node = docstring_node.prev_sibling + if get_row(docstring_node.start_point) - get_row(prev_node.end_point) <= 1: + if get_row(docstring_node.start_point) == get_row(prev_node.end_point): + # if the docstring is on the same line as the end of the classdef drop it + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + split_ds = docstring.split("\n") + docstring = "\n".join(split_ds[1:]) if len(split_ds) > 1 else "" + else: + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + self.docstring = docstring + + prop_matches = q_properties.matches(self.cls) + method_matches = q_methods.matches(self.cls) + enum_matches = q_enumerations.matches(self.cls) + event_matches = q_events.matches(self.cls) + + for _, prop_match in prop_matches: + self._parse_property_section(prop_match) + for _, enum_match in enum_matches: + self._parse_enum_section(enum_match) + for _, method_match in method_matches: + self._parse_method_section(method_match) + for _, event_match in event_matches: + self._parse_event_section(event_match) + + def _parse_property_section(self, props_match): + properties = props_match.get("properties") + if properties is None: + return + # extract property section attributes + attrs_nodes = props_match.get("attrs") + attrs = self._parse_attributes(attrs_nodes) + for prop in properties: + # match property to extract details + _, prop_match = q_property.matches(prop)[0] + # extract name (this is always available so no need for None check) + name = prop_match.get("name").text.decode( + self.encoding, errors="backslashreplace" + ) + + # extract dims list + size_type = prop_match.get("size_type") + dims_list = prop_match.get("dims") + dims = None + if dims_list is not None: + dims = tuple( + [ + dim.text.decode(self.encoding, errors="backslashreplace") + for dim in dims_list + ] + ) + elif size_type is None: + dims = None + elif size_type.text == b"scalar": + dims = ("1", "1") + elif size_type.text == b"vector": + dims = (":", "1") + elif size_type.text == b"matrix": + dims = (":", ":") + + # extract validator functions + vf_list = prop_match.get("validation_functions") + vfs = None + if vf_list is not None: + vfs = [ + vf.text.decode(self.encoding, errors="backslashreplace") + for vf in vf_list + ] + + # extract type + type_node = prop_match.get("type") + typename = ( + type_node.text.decode(self.encoding, errors="backslashreplace") + if type_node is not None + else None + ) + + # extract default + default_node = prop_match.get("default") + default = ( + process_default(default_node, self.encoding) + if default_node is not None + else None + ) + + # extract inline or following docstring if there is no semicolon + docstring_node = prop_match.get("docstring") + docstring = "" + if docstring_node is not None: + # tree-sitter-matlab combines inline comments with following + # comments which means this requires some relatively ugly + # processing, but worth it for the ease of the rest of it. + prev_sib = docstring_node.prev_named_sibling + if get_row(docstring_node.start_point) == get_row(prev_sib.end_point): + # if the docstring is on the same line as the end of the definition only take the inline part + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + docstring = docstring.split("\n")[0] + elif ( + get_row(docstring_node.start_point) - get_row(prev_sib.end_point) + <= 1 + ): + # Otherwise take the whole docstring + docstring = process_text_into_docstring( + docstring_node.text, self.encoding + ) + + # extract inline or following docstring if there _is_ a semicolon. + # this is only done if we didn't already find a docstring with the previous approach + next_node = prop.next_named_sibling + if next_node is None or docstring != "": + # Nothing to be done. + pass + elif next_node.type == "comment": + if get_row(next_node.start_point) == get_row(prop.end_point): + # if the docstring is on the same line as the end of the definition only take the inline part + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + docstring = docstring.split("\n")[0] + elif get_row(next_node.start_point) - get_row(prop.end_point) <= 1: + # Otherwise take the whole docstring + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + + # override docstring with prior if exists + prev_node = prop.prev_named_sibling + if prev_node is None: + # Nothing we can do, no previous comment + pass + elif prev_node.type == "comment": + # We have a previous comment if it ends on the previous + # line then we set the docstring. We also need to check + # if the first line of the comment is the same as a + # previous property. + if get_row(prop.start_point) - get_row(prev_node.end_point) <= 1: + ds = process_text_into_docstring(prev_node.text, self.encoding) + prev_prop = prev_node.prev_named_sibling + if prev_prop is not None and prev_prop.type == "property": + if get_row(prev_node.start_point) == get_row( + prev_prop.end_point + ): + ds = "\n".join(ds.split("\n")[1:]) + + if ds: + docstring = ds + else: + if get_row(prop.start_point) - get_row(prev_node.end_point) <= 1: + docstring = process_text_into_docstring( + prev_node.text, self.encoding + ) + elif prev_node.type == "property": + # The previous property node may have eaten our comment + # check for it a trailing comment. If it is not there + # then we stop looking. + prev_comment = prev_node.named_children[-1] + if prev_comment.type == "comment": + # we now need to check if prev_comment ends on the line + # before ours and trim the first line if it on the same + # line as prev property. + if get_row(prop.start_point) - get_row(prev_comment.end_point) <= 1: + ds = process_text_into_docstring( + prev_comment.text, self.encoding + ) + if get_row(prev_comment.start_point) == get_row( + prev_comment.prev_named_sibling.end_point + ): + ds = "\n".join(ds.split("\n")[1:]) + if ds: + docstring = ds + # After all that if our docstring is empty then we have none + if not docstring.strip(): + docstring = None + else: + pass # docstring = docstring.rstrip() + + self.properties[name] = { + "attrs": attrs, + "size": dims, + "type": typename, + "validators": vfs, + "default": default, + "docstring": docstring, + } + + def _parse_method_section(self, methods_match): + methods = methods_match.get("methods") + if methods is None: + return + attrs_nodes = methods_match.get("attrs") + attrs = self._parse_attributes(attrs_nodes) + for method in methods: + is_set_get = q_get_set.matches(method) + # Skip getter and setter + if len(is_set_get) > 0: + continue + parsed_function = MatFunctionParser(method, self.encoding) + self.methods[parsed_function.name] = parsed_function + self.methods[parsed_function.name].attrs = attrs + + def _parse_enum_section(self, enums_match): + enums = enums_match.get("enums") + if enums is None: + return + for enum in enums: + _, enum_match = q_enum.matches(enum)[0] + name = enum_match.get("name").text.decode( + self.encoding, errors="backslashreplace" + ) + arg_nodes = enum_match.get("args") + if arg_nodes is not None: + args = [ + arg.text.decode(self.encoding, errors="backslashreplace") + for arg in arg_nodes + ] + else: + args = None + + docstring = "" + # look forward for docstring + next_node = enum.next_named_sibling + if next_node is not None and next_node.type == "comment": + if get_row(next_node.start_point) == get_row(enum.end_point): + # if the docstring is on the same line as the end of the definition only take the inline part + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + docstring = docstring.split("\n")[0] + elif get_row(next_node.start_point) - get_row(enum.end_point) <= 1: + # Otherwise take the whole docstring + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + + # override docstring with prior if exists + prev_node = enum.prev_named_sibling + if prev_node is None: + # Nothing we can do, no previous comment + pass + elif prev_node.type == "comment": + # We have a previous comment if it ends on the previous + # line then we set the docstring. We also need to check + # if the first line of the comment is the same as a + # previous enum. + if get_row(enum.start_point) - get_row(prev_node.end_point) <= 1: + ds = process_text_into_docstring(prev_node.text, self.encoding) + prev_enum = prev_node.prev_named_sibling + if prev_enum is not None and prev_enum.type == "enum": + if get_row(prev_node.start_point) == get_row( + prev_enum.end_point + ): + ds = "\n".join(ds.split("\n")[1:]) + if ds: + docstring = ds + else: + if get_row(enum.start_point) - get_row(prev_node.end_point) <= 1: + docstring = process_text_into_docstring( + prev_node.text, self.encoding + ) + # After all that if our docstring is empty then we have none + if docstring.strip() == "": + docstring == None + else: + pass # docstring = docstring.rstrip() + + self.enumerations[name] = {"args": args, "docstring": docstring} + + def _parse_event_section(self, events_match): + attrs_nodes = events_match.get("attrs") + attrs = self._parse_attributes(attrs_nodes) + events = events_match.get("events") + if events is None: + return + for event in events: + name = event.text.decode(self.encoding, errors="backslashreplace") + + docstring = "" + # look forward for docstring + next_node = event.next_named_sibling + if next_node is not None and next_node.type == "comment": + if get_row(next_node.start_point) == get_row(event.end_point): + # if the docstring is on the same line as the end of the definition only take the inline part + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + docstring = docstring.split("\n")[0] + elif get_row(next_node.start_point) - get_row(event.end_point) <= 1: + # Otherwise take the whole docstring + docstring = process_text_into_docstring( + next_node.text, self.encoding + ) + + # override docstring with prior if exists + prev_node = event.prev_named_sibling + if prev_node is None: + # Nothing we can do, no previous comment + pass + elif prev_node.type == "comment": + # We have a previous comment if it ends on the previous + # line then we set the docstring. We also need to check + # if the first line of the comment is the same as a + # previous event. + if get_row(event.start_point) - get_row(prev_node.end_point) <= 1: + ds = process_text_into_docstring(prev_node.text, self.encoding) + prev_event = prev_node.prev_named_sibling + if prev_event is not None and prev_event.type == "identifier": + if get_row(prev_node.start_point) == get_row( + prev_event.end_point + ): + ds = "\n".join(ds.split("\n")[1:]) + if ds: + docstring = ds + else: + if get_row(event.start_point) - get_row(prev_node.end_point) <= 1: + docstring = process_text_into_docstring( + prev_node.text, self.encoding + ) + # After all that if our docstring is empty then we have none + if docstring.strip() == "": + docstring == None + else: + pass # docstring = docstring.rstrip() + + self.events[name] = {"attrs": attrs, "docstring": docstring} + + def _parse_attributes(self, attrs_nodes): + attrs = {} + if attrs_nodes is not None: + for attr_node in attrs_nodes: + _, attr_match = q_attributes.matches(attr_node)[0] + name = attr_match.get("name").text.decode( + self.encoding, errors="backslashreplace" + ) + value_node = attr_match.get("value") + rhs_node = attr_match.get("rhs") + if rhs_node is not None: + if rhs_node.type == "cell": + attrs[name] = [ + vn.text.decode(self.encoding, errors="backslashreplace") + for vn in value_node + ] + else: + attrs[name] = value_node[0].text.decode( + self.encoding, errors="backslashreplace" + ) + else: + attrs[name] = MATLAB_ATTRIBUTE_DEFAULTS.get(name) + + return attrs diff --git a/sphinxcontrib/mat_types.py b/sphinxcontrib/mat_types.py index 964b012..05aa151 100644 --- a/sphinxcontrib/mat_types.py +++ b/sphinxcontrib/mat_types.py @@ -16,7 +16,19 @@ from pygments.token import Token from zipfile import ZipFile import xml.etree.ElementTree as ET -import sphinxcontrib.mat_parser as mat_parser +from sphinxcontrib.mat_tree_sitter_parser import ( + MatClassParser, + MatFunctionParser, + MatScriptParser, + ML_LANG, +) +import tree_sitter_matlab as tsml +from tree_sitter import Language, Parser +import logging +from pathlib import Path +import cProfile +import pstats +from importlib.metadata import version logger = sphinx.util.logging.getLogger("matlab-domain") @@ -26,6 +38,7 @@ "MatFunction", "MatClass", "MatProperty", + "MatEnumeration", "MatMethod", "MatScript", "MatException", @@ -33,14 +46,6 @@ "MatApplication", ] -# MATLAB keywords that increment keyword-end pair count -MATLAB_KEYWORD_REQUIRES_END = list( - zip( - (Token.Keyword,) * 7, - ("arguments", "for", "if", "switch", "try", "while", "parfor"), - ) -) - # MATLAB attribute type dictionaries. @@ -101,12 +106,6 @@ "TestTags": list, } - -MATLAB_FUNC_BRACES_BEGIN = tuple(zip((Token.Punctuation,) * 2, ("(", "{"))) -MATLAB_FUNC_BRACES_END = tuple(zip((Token.Punctuation,) * 2, (")", "}"))) -MATLAB_PROP_BRACES_BEGIN = tuple(zip((Token.Punctuation,) * 3, ("(", "{", "["))) -MATLAB_PROP_BRACES_END = tuple(zip((Token.Punctuation,) * 3, (")", "}", "]"))) - # Dictionary containing all MATLAB entities that are found in `matlab_src_dir`. # The dictionary keys are both the full dotted path, relative to the root. # Further, "short names" are added. Example: @@ -489,45 +488,58 @@ def parse_mfile(mfile, name, path, encoding=None): # read mfile code if encoding is None: encoding = "utf-8" - with open(mfile, "r", encoding=encoding, errors="replace") as code_f: - code = code_f.read().replace("\r\n", "\n") + with open(mfile, "rb") as code_f: + code = code_f.read() full_code = code - # remove the top comment header (if there is one) from the code string - code = mat_parser.remove_comment_header(code) - code = mat_parser.remove_line_continuations(code) - code = mat_parser.fix_function_signatures(code) - - tks = list(MatlabLexer().get_tokens(code)) + # parse the file + tree_sitter_ver = tuple([int(sec) for sec in version("tree_sitter").split(".")]) + if tree_sitter_ver[1] == 21: + parser = Parser() + parser.set_language(ML_LANG) + else: + parser = Parser(ML_LANG) + tree = parser.parse(code) modname = path.replace(os.sep, ".") # module name # assume that functions and classes always start with a keyword - def isFunction(token): - return token == (Token.Keyword, "function") + def isFunction(tree): + q_is_function = ML_LANG.query( + r"""(source_file [(comment) "\n"]* (function_definition))""" + ) + matches = q_is_function.matches(tree.root_node) + if matches: + return True + else: + return False - def isClass(token): - return token == (Token.Keyword, "classdef") + def isClass(tree): + q_is_class = ML_LANG.query("(class_definition)") + matches = q_is_class.matches(tree.root_node) + if matches: + return True + else: + return False - if isClass(tks[0]): + if isClass(tree): logger.debug( "[sphinxcontrib-matlabdomain] parsing classdef %s from %s.", name, modname, ) - return MatClass(name, modname, tks) - elif isFunction(tks[0]): + return MatClass(name, modname, tree.root_node, encoding) + elif isFunction(tree): logger.debug( "[sphinxcontrib-matlabdomain] parsing function %s from %s.", name, modname, ) - return MatFunction(name, modname, tks) + return MatFunction(name, modname, tree.root_node, encoding) else: - # it's a script file retoken with header comment - tks = list(MatlabLexer().get_tokens(full_code)) - return MatScript(name, modname, tks) + return MatScript(name, modname, tree.root_node, encoding) + return None @staticmethod @@ -707,130 +719,6 @@ def getter(self, name, *defargs): return entity -class MatMixin(object): - """ - Methods to comparing and manipulating tokens in :class:`MatFunction` and - :class:`MatClass`. - """ - - def _tk_eq(self, idx, token): - """ - Returns ``True`` if token keys are the same and values are equal. - - :param idx: Index of token in :class:`MatObject`. - :type idx: int - :param token: Comparison token. - :type token: tuple - """ - return self.tokens[idx][0] is token[0] and self.tokens[idx][1] == token[1] - - def _tk_ne(self, idx, token): - """ - Returns ``True`` if token keys are not the same or values are not - equal. - - :param idx: Index of token in :class:`MatObject`. - :type idx: int - :param token: Comparison token. - :type token: tuple - """ - return self.tokens[idx][0] is not token[0] or self.tokens[idx][1] != token[1] - - def _eotk(self, idx): - """ - Returns ``True`` if end of tokens is reached. - """ - return idx >= len(self.tokens) - - def _blanks(self, idx): - """ - Returns number of blank text tokens. - - :param idx: Token index. - :type idx: int - """ - # idx0 = idx # original index - # while self._tk_eq(idx, (Token.Text, ' ')): idx += 1 - # return idx - idx0 # blanks - return self._indent(idx) - - def _whitespace(self, idx): - """ - Returns number of whitespaces text tokens, including blanks, newline - and tabs. - - :param idx: Token index. - :type idx: int - """ - idx0 = idx # original index - while ( - self.tokens[idx][0] is Token.Text - or self.tokens[idx][0] is Token.Text.Whitespace - ) and self.tokens[idx][1] in [" ", "\n", "\t"]: - idx += 1 - return idx - idx0 # whitespace - - def _indent(self, idx): - """ - Returns indentation tabs or spaces. No indentation is zero. - - :param idx: Token index. - :type idx: int - """ - idx0 = idx # original index - while self.tokens[idx][0] is Token.Text and self.tokens[idx][1] in [" ", "\t"]: - idx += 1 - return idx - idx0 # indentation - - def _propspec(self, idx): - """ - Returns number of "property" specification tokens - - :param idx: Token index. - :type idx: int - """ - idx0 = idx # original index - while ( - self._tk_eq(idx, (Token.Punctuation, "@")) - or self._tk_eq(idx, (Token.Punctuation, "(")) - or self._tk_eq(idx, (Token.Punctuation, ")")) - or self._tk_eq(idx, (Token.Punctuation, ",")) - or self._tk_eq(idx, (Token.Punctuation, ":")) - or self.tokens[idx][0] == Token.Literal.Number.Integer - or self._tk_eq(idx, (Token.Punctuation, "{")) - or self._tk_eq(idx, (Token.Punctuation, "}")) - or self._tk_eq(idx, (Token.Punctuation, "[")) - or self._tk_eq(idx, (Token.Punctuation, "]")) - or self._tk_eq(idx, (Token.Punctuation, ".")) - or self.tokens[idx][0] == Token.Literal.String - or self.tokens[idx][0] == Token.Name - or (self.tokens[idx][0] == Token.Text and self.tokens[idx][1] != "\n") - ): - idx += 1 - - count = idx - idx0 # property spec count. - propspec = "".join([content for _, content in self.tokens[idx0 : idx0 + count]]) - propspec = propspec.strip() - return count, propspec - - def _is_newline(self, idx): - """Returns true if the token at index is a newline""" - return ( - self.tokens[idx][0] in (Token.Text, Token.Text.Whitespace) - and self.tokens[idx][1] == "\n" - ) - - -def skip_whitespace(tks): - """Eats whitespace from list of tokens""" - while tks and ( - tks[-1][0] == Token.Text.Whitespace - or tks[-1][0] == Token.Text - and tks[-1][1] in [" ", "\t"] - ): - tks.pop() - - class MatFunction(MatObject): """ A MATLAB function. @@ -843,179 +731,19 @@ class MatFunction(MatObject): :type tokens: list """ - def __init__(self, name, modname, tokens): + def __init__(self, name, modname, tokens, encoding): super(MatFunction, self).__init__(name) + parsed_function = MatFunctionParser(tokens, encoding) #: Path of folder containing :class:`MatObject`. self.module = modname - #: List of tokens parsed from mfile by Pygments. - self.tokens = tokens #: docstring - self.docstring = "" + self.docstring = parsed_function.docstring #: output args - self.retv = None + self.retv = parsed_function.retv #: input args - self.args = None + self.args = parsed_function.args #: remaining tokens after main function is parsed self.rem_tks = None - # ===================================================================== - # parse tokens - # XXX: Pygments always reads MATLAB function signature as: - # [(Token.Keyword, 'function'), # any whitespace is stripped - # (Token.Text.Whitesapce, ' '), # spaces and tabs are concatenated - # (Token.Text, '[o1, o2]'), # if there are outputs, they're all - # concatenated w/ or w/o brackets and any - # trailing whitespace - # (Token.Punctuation, '='), # possibly an equal sign - # (Token.Text.Whitesapce, ' '), # spaces and tabs are concatenated - # (Token.Name.Function, 'myfun'), # the name of the function - # (Token.Punctuation, '('), # opening parenthesis - # (Token.Text, 'a1, a2', # if there are args, they're concatenated - # (Token.Punctuation, ')'), # closing parenthesis - # (Token.Text.Whitesapce, '\n')] # all whitespace after args - # XXX: Pygments does not tolerate MATLAB continuation ellipsis! - tks = copy(self.tokens) # make a copy of tokens - tks.reverse() # reverse in place for faster popping, stacks are LiLo - try: - # ===================================================================== - # parse function signature - # function [output] = name(inputs) - # % docstring - # ===================================================================== - # Skip function token - already checked in MatObject.parse_mfile - tks.pop() - skip_whitespace(tks) - - # Check for return values - retv = tks.pop() - if retv[0] is Token.Text: - self.retv = [rv.strip() for rv in retv[1].strip("[ ]").split(",")] - if len(self.retv) == 1: - # check if return is empty - if not self.retv[0]: - self.retv = None - # check if return delimited by whitespace - elif " " in self.retv[0] or "\t" in self.retv[0]: - self.retv = [ - rv - for rv_tab in self.retv[0].split("\t") - for rv in rv_tab.split(" ") - ] - if tks.pop() != (Token.Punctuation, "="): - # Unlikely to end here. But never-the-less warn! - logger.warning( - "[sphinxcontrib-matlabdomain] Parsing failed in %s.%s. Expected '='.", - modname, - name, - ) - return - - skip_whitespace(tks) - elif retv[0] is Token.Name.Function: - tks.append(retv) - # ===================================================================== - # function name - func_name = tks.pop() - func_name = ( - func_name[0], - func_name[1].strip(" ()"), - ) # Strip () in case of dummy arg - if func_name != (Token.Name.Function, self.name): # @UndefinedVariable - if isinstance(self, MatMethod): - self.name = func_name[1] - else: - logger.warning( - "[sphinxcontrib-matlabdomain] Unexpected function name: '%s'. " - "Expected '%s' in module '%s'.", - func_name[1], - name, - modname, - ) - - # ===================================================================== - # input args - if tks.pop() == (Token.Punctuation, "("): - args = tks.pop() - if args[0] is Token.Text: - self.args = [ - arg.strip() for arg in args[1].split(",") - ] # no arguments given - elif args == (Token.Punctuation, ")"): - # put closing parenthesis back in stack - tks.append(args) - # check if function args parsed correctly - if tks.pop() != (Token.Punctuation, ")"): - # Unlikely to end here. But never-the-less warn! - logger.warning( - "[sphinxcontrib-matlabdomain] Parsing failed in {}.{}. Expected ')'.", - modname, - name, - ) - return - - skip_whitespace(tks) - # ===================================================================== - # docstring - try: - docstring = tks.pop() - except IndexError: - docstring = None - while docstring and docstring[0] is Token.Comment: - self.docstring += docstring[1].lstrip("%") - # Get newline if it exists and append to docstring - try: - wht = tks.pop() # We expect a newline - except IndexError: - break - if wht[0] in (Token.Text, Token.Text.Whitespace) and wht[1] == "\n": - self.docstring += "\n" - # Skip whitespace - try: - wht = tks.pop() # We expect a newline - except IndexError: - break - while wht in list(zip((Token.Text,) * 3, (" ", "\t"))): - try: - wht = tks.pop() - except IndexError: - break - docstring = wht # check if Token is Comment - - # Find the end of the function - used in `MatMethod`` to determine where a method ends. - if docstring is None: - return - kw = docstring # last token - lastkw = 0 # set last keyword placeholder - kw_end = 1 # count function keyword - while kw_end > 0: - # increment keyword-end pairs count - if kw in MATLAB_KEYWORD_REQUIRES_END: - kw_end += 1 - # nested function definition - elif kw[0] is Token.Keyword and kw[1].strip() == "function": - kw_end += 1 - # decrement keyword-end pairs count but - # don't decrement `end` if used as index - elif kw == (Token.Keyword, "end") and not lastkw: - kw_end -= 1 - # save last punctuation - elif kw in MATLAB_FUNC_BRACES_BEGIN: - lastkw += 1 - elif kw in MATLAB_FUNC_BRACES_END: - lastkw -= 1 - try: - kw = tks.pop() - except IndexError: - break - tks.append(kw) # put last token back in list - except IndexError: - logger.warning( - "[sphinxcontrib-matlabdomain] Parsing failed in %s.%s. Check if valid MATLAB code.", - modname, - name, - ) - # if there are any tokens left save them - if len(tks) > 0: - self.rem_tks = tks # save extra tokens def ref_role(self): """Returns role to use for references to this object (e.g. when generating auto-links)""" @@ -1040,7 +768,7 @@ def getter(self, name, *defargs): super(MatFunction, self).getter(name, *defargs) -class MatClass(MatMixin, MatObject): +class MatClass(MatObject): """ A MATLAB class definition. @@ -1052,392 +780,28 @@ class MatClass(MatMixin, MatObject): :type tokens: list """ - def __init__(self, name, modname, tokens): + def __init__(self, name, modname, tokens, encoding): super(MatClass, self).__init__(name) + parsed_class = MatClassParser(tokens, encoding) #: Path of folder containing :class:`MatObject`. self.module = modname - #: List of tokens parsed from mfile by Pygments. - self.tokens = tokens #: dictionary of class attributes - self.attrs = {} + self.attrs = parsed_class.attrs #: list of class superclasses - self.bases = [] + self.bases = parsed_class.supers #: docstring - self.docstring = "" + self.docstring = parsed_class.docstring #: dictionary of class properties - self.properties = {} + self.properties = parsed_class.properties #: dictionary of class methods - self.methods = {} + self.methods = { + name: MatMethod(name, parsed_fun, modname, self) + for (name, parsed_fun) in parsed_class.methods.items() + } + #: + self.enumerations = parsed_class.enumerations #: remaining tokens after main class definition is parsed self.rem_tks = None - # ===================================================================== - # parse tokens - # TODO: use generator and next() instead of stepping index! - try: - # Skip classdef token - already checked in MatObject.parse_mfile - idx = 1 # token index - - # class "attributes" - self.attrs, idx = self.attributes(idx, MATLAB_CLASS_ATTRIBUTE_TYPES) - - # Check if self.name matches the name in the file. - idx += self._blanks(idx) - if not self.tokens[idx][1] == self.name: - logger.warning( - "[sphinxcontrib-matlabdomain] Unexpected class name: '%s'." - " Expected '%s' in '%s'.", - self.tokens[idx][1], - name, - modname, - ) - - idx += 1 - idx += self._blanks(idx) # skip blanks - # ===================================================================== - # super classes - if self._tk_eq(idx, (Token.Operator, "<")): - idx += 1 - # newline terminates superclasses - while not self._is_newline(idx): - idx += self._blanks(idx) # skip blanks - # concatenate base name - base_name = "" - while ( - not self._whitespace(idx) - and self.tokens[idx][0] is not Token.Comment - ): - base_name += self.tokens[idx][1] - idx += 1 - # If it's a newline, we are done parsing. - if not self._is_newline(idx): - idx += 1 - if base_name: - self.bases.append(base_name) - idx += self._blanks(idx) # skip blanks - # continue to next super class separated by & - if self._tk_eq(idx, (Token.Operator, "&")): - idx += 1 - idx += 1 # end of super classes - # newline terminates classdef signature - elif self._is_newline(idx): - idx += 1 # end of classdef signature - # ===================================================================== - # docstring - idx += self._indent(idx) # calculation indentation - # concatenate docstring - while self.tokens[idx][0] is Token.Comment: - self.docstring += self.tokens[idx][1].lstrip("%") - idx += 1 - # append newline to docstring - if self._is_newline(idx): - self.docstring += self.tokens[idx][1] - idx += 1 - # skip tab - indent = self._indent(idx) # calculation indentation - idx += indent - # ===================================================================== - # properties & methods blocks - # loop over code body searching for blocks until end of class - while self._tk_ne(idx, (Token.Keyword, "end")): - # skip comments and whitespace - while self._whitespace(idx) or self.tokens[idx][0] is Token.Comment: - whitespace = self._whitespace(idx) - if whitespace: - idx += whitespace - else: - idx += 1 - - # ================================================================= - # properties blocks - if self._tk_eq(idx, (Token.Keyword, "properties")): - prop_name = "" - idx += 1 - # property "attributes" - attr_dict, idx = self.attributes( - idx, MATLAB_PROPERTY_ATTRIBUTE_TYPES - ) - # Token.Keyword: "end" terminates properties & methods block - while self._tk_ne(idx, (Token.Keyword, "end")): - # skip whitespace - while self._whitespace(idx): - whitespace = self._whitespace(idx) - if whitespace: - idx += whitespace - else: - idx += 1 - - # ========================================================= - # long docstring before property - if self.tokens[idx][0] is Token.Comment: - # docstring - docstring = "" - - # Collect comment lines - while self.tokens[idx][0] is Token.Comment: - docstring += self.tokens[idx][1].lstrip("%") - idx += 1 - idx += self._blanks(idx) - - try: - # Check if end of line was reached - if self._is_newline(idx): - docstring += "\n" - idx += 1 - idx += self._blanks(idx) - - # Check if variable name is next - if self.tokens[idx][0] is Token.Name: - prop_name = self.tokens[idx][1] - self.properties[prop_name] = { - "attrs": attr_dict - } - self.properties[prop_name][ - "docstring" - ] = docstring - break - - # If there is an empty line at the end of - # the comment: discard it - elif self._is_newline(idx): - docstring = "" - idx += self._whitespace(idx) - break - - except IndexError: - # EOF reached, quit gracefully - break - - # with "%:" directive trumps docstring after property - isTokenName = self.tokens[idx][0] is Token.Name - isTokenNameSubtype = self.tokens[idx][0] in Token.Name.subtypes - if isTokenName or isTokenNameSubtype: - prop_name = self.tokens[idx][1] - idx += 1 - if isTokenNameSubtype: - logger.debug( - "[sphinxcontrib-matlabdomain] WARNING %s.%s.%s is a builtin name.", - self.module, - self.name, - prop_name, - ) - - # Initialize property if it was not already done - if prop_name not in self.properties.keys(): - self.properties[prop_name] = {"attrs": attr_dict} - - # Capture (dimensions) class {validators} as "specs" - # https://mathworks.com/help/matlab/matlab_oop/defining-properties.html - count, propspec = self._propspec(idx) - self.properties[prop_name]["specs"] = propspec - - idx = idx + count - if self._tk_eq(idx, (Token.Punctuation, ";")): - continue - - elif self._tk_eq(idx, (Token.Keyword, "end")): - idx += 1 - break - # skip semicolon after property name, but no default - elif self._tk_eq(idx, (Token.Punctuation, ";")): - idx += 1 - # A comment might come after semi-colon - idx += self._blanks(idx) - if self._is_newline(idx): - idx += 1 - # Property definition is finished; add missing values - if "default" not in self.properties[prop_name].keys(): - self.properties[prop_name]["default"] = None - if "docstring" not in self.properties[prop_name].keys(): - self.properties[prop_name]["docstring"] = None - - continue - elif self.tokens[idx][0] is Token.Comment: - docstring = self.tokens[idx][1].lstrip("%") - docstring += "\n" - self.properties[prop_name]["docstring"] = docstring - idx += 1 - elif self.tokens[idx][0] is Token.Comment: - # Comments seperated with blank lines. - idx = idx - 1 - continue - else: - logger.warning( - "sphinxcontrib-matlabdomain] Expected property in %s.%s - got %s", - self.module, - self.name, - str(self.tokens[idx]), - ) - return - idx += self._blanks(idx) # skip blanks - # ========================================================= - # defaults - default = {"default": None} - if self._tk_eq(idx, (Token.Punctuation, "=")): - idx += 1 - idx += self._blanks(idx) # skip blanks - # concatenate default value until newline or comment - default = "" - brace_count = 0 - # keep reading until newline or comment - # only if all punctuation pairs are closed - # and comment is **not** continuation ellipsis - while ( - ( - not self._is_newline(idx) - and self.tokens[idx][0] is not Token.Comment - ) - or brace_count > 0 - or ( - self.tokens[idx][0] is Token.Comment - and self.tokens[idx][1].startswith("...") - ) - ): - token = self.tokens[idx] - # default has an array spanning multiple lines - # keep track of braces - if token in MATLAB_PROP_BRACES_BEGIN: - brace_count += 1 - # look for end of array - elif token in MATLAB_PROP_BRACES_END: - brace_count -= 1 - # Pygments treats continuation ellipsis as comments - # text from ellipsis until newline is in token - elif token[0] is Token.Comment and token[1].startswith( - "..." - ): - idx += 1 # skip ellipsis comments - # include newline which should follow comment - if self._is_newline(idx): - default += "\n" - idx += 1 - continue - elif self._is_newline(idx - 1) and not self._is_newline( - idx - ): - idx += self._blanks(idx) - continue - elif token[0] is Token.Text and token[1] == " ": - # Skip spaces that are not in strings. - idx += 1 - continue - default += token[1] - idx += 1 - if self.tokens[idx][0] is not Token.Comment: - idx += 1 - if default: - default = {"default": default.rstrip("; ")} - - self.properties[prop_name].update(default) - # ========================================================= - # docstring - if "docstring" not in self.properties[prop_name].keys(): - docstring = {"docstring": None} - if self.tokens[idx][0] is Token.Comment: - docstring["docstring"] = self.tokens[idx][1].lstrip("%") - idx += 1 - self.properties[prop_name].update(docstring) - elif self.tokens[idx][0] is Token.Comment: - # skip this comment - idx += 1 - - idx += self._whitespace(idx) - idx += 1 - # ================================================================= - # method blocks - if self._tk_eq(idx, (Token.Keyword, "methods")): - idx += 1 - # method "attributes" - attr_dict, idx = self.attributes(idx, MATLAB_METHOD_ATTRIBUTE_TYPES) - # Token.Keyword: "end" terminates properties & methods block - while self._tk_ne(idx, (Token.Keyword, "end")): - # skip comments and whitespace - while ( - self._whitespace(idx) - or self.tokens[idx][0] is Token.Comment - ): - whitespace = self._whitespace(idx) - if whitespace: - idx += whitespace - else: - idx += 1 - # skip methods defined in other files - meth_tk = self.tokens[idx] - if ( - meth_tk[0] is Token.Name - or meth_tk[0] is Token.Name.Builtin - or meth_tk[0] is Token.Name.Function - or ( - meth_tk[0] is Token.Keyword - and meth_tk[1].strip() == "function" - and self.tokens[idx + 1][0] is Token.Name.Function - ) - or self._tk_eq(idx, (Token.Punctuation, "[")) - or self._tk_eq(idx, (Token.Punctuation, "]")) - or self._tk_eq(idx, (Token.Punctuation, "=")) - or self._tk_eq(idx, (Token.Punctuation, "(")) - or self._tk_eq(idx, (Token.Punctuation, ")")) - or self._tk_eq(idx, (Token.Punctuation, ";")) - or self._tk_eq(idx, (Token.Punctuation, ",")) - ): - logger.debug( - "[sphinxcontrib-matlabdomain] Skipping tokens for methods defined in separate files." - "Token #%d: %r", - idx, - self.tokens[idx], - ) - idx += 1 + self._whitespace(idx + 1) - elif self._tk_eq(idx, (Token.Keyword, "end")): - idx += 1 - break - else: - # find methods - meth = MatMethod( - self.module, self.tokens[idx:], self, attr_dict - ) - - # Detect getter/setter methods - these are not documented - isGetter = meth.name.startswith("get.") - isSetter = meth.name.startswith("set.") - if not (isGetter or isSetter): - # Add the parsed method to methods dictionary - self.methods[meth.name] = meth - - # Update idx with the number of parsed tokens. - idx += meth.skip_tokens() - idx += self._whitespace(idx) - idx += 1 - if self._tk_eq(idx, (Token.Keyword, "events")): - logger.debug( - "[sphinxcontrib-matlabdomain] ignoring 'events' in 'classdef %s.'", - self.name, - ) - idx += 1 - # Token.Keyword: "end" terminates events block - while self._tk_ne(idx, (Token.Keyword, "end")): - idx += 1 - idx += 1 - if self._tk_eq(idx, (Token.Name, "enumeration")): - logger.debug( - "[sphinxcontrib-matlabdomain] ignoring 'enumeration' in 'classdef %s'.", - self.name, - ) - idx += 1 - # Token.Keyword: "end" terminates events block - while self._tk_ne(idx, (Token.Keyword, "end")): - idx += 1 - idx += 1 - if self._tk_eq(idx, (Token.Punctuation, ";")): - # Skip trailing semicolon after end. - idx += 1 - except IndexError: - logger.warning( - "[sphinxcontrib-matlabdomain] Parsing failed in %s.%s. " - "Check if valid MATLAB code.", - modname, - name, - ) - - self.rem_tks = idx # index of last token def ref_role(self): """Returns role to use for references to this object (e.g. when generating auto-links)""" @@ -1468,105 +832,6 @@ def link(self, env, name=None): else: return f":class:`{target}`" - def attributes(self, idx, attr_types): - """ - Retrieve MATLAB class, property and method attributes. - """ - attr_dict = {} - idx += self._blanks(idx) # skip blanks - # class, property & method "attributes" start with parenthesis - if self._tk_eq(idx, (Token.Punctuation, "(")): - idx += 1 - # closing parenthesis terminates attributes - while self._tk_ne(idx, (Token.Punctuation, ")")): - idx += self._blanks(idx) # skip blanks - - k, attr_name = self.tokens[idx] # split token key, value - if k is Token.Name and attr_name in attr_types: - attr_dict[attr_name] = True # add attibute to dictionary - idx += 1 - elif k is Token.Name: - logger.warning( - "[sphinxcontrib-matlabdomain] Unexpected class attribute: '%s'. " - " In '%s.%s'.", - str(self.tokens[idx][1]), - self.module, - self.name, - ) - idx += 1 - - idx += self._blanks(idx) # skip blanks - - # Continue if attribute is assigned a boolean value - if self.tokens[idx][0] == Token.Name.Builtin: - idx += 1 - continue - - # continue to next attribute separated by commas - if self._tk_eq(idx, (Token.Punctuation, ",")): - idx += 1 - continue - # attribute values - elif self._tk_eq(idx, (Token.Punctuation, "=")): - idx += 1 - idx += self._blanks(idx) # skip blanks - k, attr_val = self.tokens[idx] # split token key, value - if k is Token.Name and attr_val in ["true", "false"]: - # logical value - if attr_val == "false": - attr_dict[attr_name] = False - idx += 1 - elif k is Token.Name or self._tk_eq(idx, (Token.Text, "?")): - # concatenate enumeration or meta class - enum_or_meta = self.tokens[idx][1] - idx += 1 - while ( - self._tk_ne(idx, (Token.Text, " ")) - and self._tk_ne(idx, (Token.Text, "\t")) - and self._tk_ne(idx, (Token.Punctuation, ",")) - and self._tk_ne(idx, (Token.Punctuation, ")")) - ): - enum_or_meta += self.tokens[idx][1] - idx += 1 - if self._tk_ne(idx, (Token.Punctuation, ")")): - idx += 1 - attr_dict[attr_name] = enum_or_meta - # cell array of values - elif self._tk_eq(idx, (Token.Punctuation, "{")): - idx += 1 - # closing curly braces terminate cell array - attr_dict[attr_name] = [] - while self._tk_ne(idx, (Token.Punctuation, "}")): - idx += self._blanks(idx) # skip blanks - # concatenate attr value string - attr_val = "" - # TODO: use _blanks or _indent instead - while self._tk_ne( - idx, (Token.Punctuation, ",") - ) and self._tk_ne(idx, (Token.Punctuation, "}")): - attr_val += self.tokens[idx][1] - idx += 1 - if self._tk_eq(idx, (Token.Punctuation, ",")): - idx += 1 - if attr_val: - attr_dict[attr_name].append(attr_val) - idx += 1 - elif ( - self.tokens[idx][0] == Token.Literal.String - and self.tokens[idx + 1][0] == Token.Literal.String - ): - # String - attr_val += self.tokens[idx][1] + self.tokens[idx + 1][1] - idx += 2 - attr_dict[attr_name] = attr_val.strip("'") - - idx += self._blanks(idx) # skip blanks - # continue to next attribute separated by commas - if self._tk_eq(idx, (Token.Punctuation, ",")): - idx += 1 - idx += 1 # end of class attributes - return attr_dict, idx - @property def __module__(self): return self.module @@ -1577,13 +842,13 @@ def __doc__(self): @property def __bases__(self): - bases_ = dict.fromkeys(self.bases) # make copy of bases + bases_ = dict.fromkeys([base for base in self.bases]) # make copy of bases class_entity_table = {} for name, entity in entities_table.items(): if isinstance(entity, MatClass) or "@" in name: class_entity_table[name] = entity - for base in self.bases: + for base in bases_.keys(): if base in class_entity_table.keys(): bases_[base] = class_entity_table[base] @@ -1603,11 +868,18 @@ def getter(self, name, *defargs): return self.__bases__ elif name in self.properties: return MatProperty(name, self, self.properties[name]) + elif name in self.enumerations: + return MatEnumeration(name, self, self.enumerations[name]) elif name in self.methods: return self.methods[name] + elif name in self.enumerations: + return elif name == "__dict__": objdict = dict([(pn, self.getter(pn)) for pn in self.properties.keys()]) objdict.update(self.methods) + objdict.update( + dict([(en, self.getter(en)) for en in self.enumerations.keys()]) + ) return objdict else: super(MatClass, self).getter(name, *defargs) @@ -1620,7 +892,9 @@ def __init__(self, name, cls, attrs): self.attrs = attrs["attrs"] self.default = attrs["default"] self.docstring = attrs["docstring"] - self.specs = attrs["specs"] + self.size = attrs["size"] + self.type = attrs["type"] + self.validators = attrs["validators"] def ref_role(self): """Returns role to use for references to this object (e.g. when generating auto-links)""" @@ -1635,25 +909,43 @@ def __doc__(self): return self.docstring +class MatEnumeration(MatObject): + def __init__(self, name, cls, attrs): + super(MatEnumeration, self).__init__(name) + self.cls = cls + self.docstring = attrs["docstring"] + + def ref_role(self): + """Returns role to use for references to this object (e.g. when generating auto-links)""" + return "enum" + + @property + def __module__(self): + return self.cls.module + + @property + def __doc__(self): + return self.docstring + + class MatMethod(MatFunction): - def __init__(self, modname, tks, cls, attrs): - # set name to None - super(MatMethod, self).__init__(None, modname, tks) + def __init__(self, name, parsed_function, modname, cls): + self.name = name + #: Path of folder containing :class:`MatObject`. + self.module = modname + #: docstring + self.docstring = parsed_function.docstring + #: output args + self.retv = parsed_function.retv + #: input args + self.args = parsed_function.args self.cls = cls - self.attrs = attrs + self.attrs = parsed_function.attrs def ref_role(self): """Returns role to use for references to this object (e.g. when generating auto-links)""" return "meth" - def skip_tokens(self): - # Number of tokens to skip in `MatClass` - num_rem_tks = len(self.rem_tks) - len_meth = len(self.tokens) - num_rem_tks - self.tokens = self.tokens[:-num_rem_tks] - self.rem_tks = None - return len_meth - @property def __module__(self): return self.module @@ -1664,49 +956,15 @@ def __doc__(self): class MatScript(MatObject): - def __init__(self, name, modname, tks): + def __init__(self, name, modname, tks, encoding): super(MatScript, self).__init__(name) + parsed_script = MatScriptParser(tks, encoding) #: Path of folder containing :class:`MatScript`. self.module = modname #: List of tokens parsed from mfile by Pygments. self.tokens = tks #: docstring - self.docstring = "" - #: remaining tokens after main function is parsed - self.rem_tks = None - - tks = copy(self.tokens) # make a copy of tokens - tks.reverse() # reverse in place for faster popping, stacks are LiLo - skip_whitespace(tks) - # ===================================================================== - # docstring - try: - docstring = tks.pop() - # Skip any statements before first documentation header - while docstring and docstring[0] is not Token.Comment: - docstring = tks.pop() - except IndexError: - docstring = None - while docstring and docstring[0] is Token.Comment: - self.docstring += docstring[1].lstrip("%") - # Get newline if it exists and append to docstring - try: - wht = tks.pop() # We expect a newline - except IndexError: - break - if wht[0] in (Token.Text, Token.Text.Whitespace) and wht[1] == "\n": - self.docstring += "\n" - # Skip whitespace - try: - wht = tks.pop() # We expect a newline - except IndexError: - break - while wht in list(zip((Token.Text,) * 3, (" ", "\t"))): - try: - wht = tks.pop() - except IndexError: - break - docstring = wht # check if Token is Comment + self.docstring = parsed_script.docstring @property def __doc__(self): diff --git a/sphinxcontrib/matlab.py b/sphinxcontrib/matlab.py index 43b96ab..c95463a 100644 --- a/sphinxcontrib/matlab.py +++ b/sphinxcontrib/matlab.py @@ -336,6 +336,24 @@ class MatClasslike(MatObject): Description of a class-like object (classes, interfaces, exceptions). """ + def _object_hierarchy_parts(self, sig): + """ + Returns a tuple of strings, one entry for each part of the object's + hierarchy (e.g. ``('module', 'submodule', 'Class', 'method')``). The + returned tuple is used to properly nest children within parents in the + table of contents, and can also be used within the + :py:meth:`_toc_entry_name` method. + + This method must not be used without table of contents generation. + """ + parts = sig.attributes.get("module").split(".") + parts.append(sig.attributes.get("fullname")) + return tuple(parts) + + def _toc_entry_name(self, sig): + # TODO respecting the configuration setting ``toc_object_entries_show_parents`` + return sig.attributes.get("fullname") + def get_signature_prefix(self, sig): return self.objtype + " " @@ -693,6 +711,7 @@ class MATLABDomain(Domain): "class": MatXRefRole(), "const": MatXRefRole(), "attr": MatXRefRole(), + "enum": MatXRefRole(), "meth": MatXRefRole(fix_parens=True), "mod": MatXRefRole(), "obj": MatXRefRole(), @@ -902,6 +921,11 @@ def setup(app): "mat", "autoattribute", mat_directives.MatlabAutodocDirective ) + app.registry.add_documenter("mat:enum", doc.MatAttributeDocumenter) + app.add_directive_to_domain( + "mat", "autoenum", mat_directives.MatlabAutodocDirective + ) + app.registry.add_documenter("mat:data", doc.MatDataDocumenter) app.add_directive_to_domain( "mat", "autodata", mat_directives.MatlabAutodocDirective diff --git a/tests/test_data/f_with_input_argument_block.m b/tests/test_data/f_with_input_argument_block.m new file mode 100644 index 0000000..5b191c0 --- /dev/null +++ b/tests/test_data/f_with_input_argument_block.m @@ -0,0 +1,12 @@ +function [o1, o2, o3] = f_with_input_argument_block(a1, a2) + arguments + a1(1,1) double = 0 % the first input + a2(1,1) double = a1 % another input + end + o1 = a1; o2 = a2; o3 = a1 + a2; + for n = 1:3 + o1 = o2; + o2 = o3; + o3 = o1 + o2; + end +end diff --git a/tests/test_data/f_with_output_argument_block.m b/tests/test_data/f_with_output_argument_block.m new file mode 100644 index 0000000..e063f7b --- /dev/null +++ b/tests/test_data/f_with_output_argument_block.m @@ -0,0 +1,13 @@ +function [o1, o2, o3] = f_with_output_argument_block(a1, a2) + arguments(Output) + o1(1,1) double % Output one + o2(1,:) double % Another output + o3(1,1) double {mustBePositive} % A third output + end + o1 = a1; o2 = a2; o3 = a1 + a2; + for n = 1:3 + o1 = o2; + o2 = o3; + o3 = o1 + o2; + end +end diff --git a/tests/test_matlabify.py b/tests/test_matlabify.py index 1e54fcc..42ed9a8 100644 --- a/tests/test_matlabify.py +++ b/tests/test_matlabify.py @@ -99,6 +99,8 @@ def test_module(mod): "ClassWithEnumMethod", "ClassWithEventMethod", "f_with_function_variable", + "f_with_input_argument_block", + "f_with_output_argument_block", "ClassWithUndocumentedMembers", "ClassWithGetterSetter", "ClassWithDoubleQuotedString", @@ -141,9 +143,16 @@ def test_classes(mod): assert cls.bases == ["handle", "my.super.Class"] assert cls.attrs == {} assert cls.properties == { - "x": {"attrs": {}, "default": None, "docstring": " a property", "specs": ""} + "x": { + "attrs": {}, + "default": None, + "docstring": "a property", + "size": None, + "type": None, + "validators": None, + } } - assert cls.getter("__doc__") == " a handle class\n\n :param x: a variable\n" + assert cls.getter("__doc__") == "a handle class\n\n:param x: a variable" def test_abstract_class(mod): @@ -159,33 +168,37 @@ def test_abstract_class(mod): assert abc.properties == { "y": { "default": None, - "docstring": " y variable", + "docstring": "y variable", "attrs": {"GetAccess": "private", "SetAccess": "private"}, - "specs": "", + "size": None, + "type": None, + "validators": None, }, "version": { "default": "'0.1.1-beta'", - "docstring": " version", + "docstring": "version", "attrs": {"Constant": True}, - "specs": "", + "size": None, + "type": None, + "validators": None, }, } assert ( abc.getter("__doc__") - == " an abstract class\n\n :param y: a variable\n :type y: double\n" + == "an abstract class\n\n:param y: a variable\n:type y: double" ) assert abc.getter("__doc__") == abc.docstring abc_y = abc.getter("y") assert isinstance(abc_y, doc.MatProperty) assert abc_y.default is None - assert abc_y.docstring == " y variable" + assert abc_y.docstring == "y variable" assert abc_y.attrs == {"SetAccess": "private", "GetAccess": "private"} abc_version = abc.getter("version") assert isinstance(abc_version, doc.MatProperty) assert abc_version.default == "'0.1.1-beta'" - assert abc_version.docstring == " version" + assert abc_version.docstring == "version" assert abc_version.attrs == {"Constant": True} @@ -195,7 +208,7 @@ def test_class_method(mod): assert cls_meth.getter("__name__") == "ClassExample" assert ( cls_meth.docstring - == " test class methods\n\n :param a: the input to :class:`ClassExample`\n" + == "test class methods\n\n:param a: the input to :class:`ClassExample`" ) constructor = cls_meth.getter("ClassExample") assert isinstance(constructor, doc.MatMethod) @@ -206,18 +219,18 @@ def test_class_method(mod): # TODO: mymethod.args will contain ['obj', 'b'] if run standalone # but if test_autodoc.py is run, the 'obj' is removed assert mymethod.args - assert mymethod.args[-1] == "b" - assert mymethod.retv == ["c"] + assert "b" in list(mymethod.args.keys()) + assert list(mymethod.retv.keys()) == ["c"] assert ( mymethod.docstring - == " a method in :class:`ClassExample`\n\n :param b: an input to :meth:`mymethod`\n" + == "a method in :class:`ClassExample`\n\n:param b: an input to :meth:`mymethod`" ) def test_submodule_class(mod): cls = mod.getter("submodule.TestFibonacci") assert isinstance(cls, doc.MatClass) - assert cls.docstring == " Test of MATLAB unittest method attributes\n" + assert cls.docstring == "Test of MATLAB unittest method attributes" assert cls.attrs == {} assert cls.bases == ["matlab.unittest.TestCase"] assert "compareFirstThreeElementsToExpected" in cls.methods @@ -226,17 +239,17 @@ def test_submodule_class(mod): method = cls.getter("compareFirstThreeElementsToExpected") assert isinstance(method, doc.MatMethod) assert method.name == "compareFirstThreeElementsToExpected" - assert method.retv is None - assert method.args == ["tc"] - assert method.docstring == " Test case that compares first three elements\n" - assert method.attrs == {"Test": True} + assert method.retv == {} + assert list(method.args.keys()) == ["tc"] + assert method.docstring == "Test case that compares first three elements" + assert method.attrs == {"Test": None} def test_folder_class(mod): cls_mod = mod.getter("@ClassFolder") assert isinstance(cls_mod, doc.MatModule) cls = cls_mod.getter("ClassFolder") - assert cls.docstring == " A class in a folder\n" + assert cls.docstring == "A class in a folder" assert cls.attrs == {} assert cls.bases == [] assert cls.module == "test_data.@ClassFolder" @@ -244,8 +257,10 @@ def test_folder_class(mod): "p": { "attrs": {}, "default": None, - "docstring": " a property of a class folder", - "specs": "", + "docstring": "a property of a class folder", + "size": None, + "type": None, + "validators": None, } } @@ -254,18 +269,18 @@ def test_folder_class(mod): func = cls_mod.getter("a_static_func") assert isinstance(func, doc.MatFunction) assert func.name == "a_static_func" - assert func.args == ["args"] - assert func.retv == ["retv"] - assert func.docstring == " method in :class:`~test_data.@ClassFolder`\n" + assert list(func.args.keys()) == ["args"] + assert list(func.retv.keys()) == ["retv"] + assert func.docstring == "method in :class:`~test_data.@ClassFolder`" func = cls_mod.getter("classMethod") assert isinstance(func, doc.MatFunction) assert func.name == "classMethod" - assert func.args == ["obj", "varargin"] - assert func.retv == ["varargout"] + assert list(func.args.keys()) == ["obj", "varargin"] + assert list(func.retv.keys()) == ["varargout"] assert ( func.docstring - == " CLASSMETHOD A function within a package\n\n :param obj: An instance of this class.\n" - " :param varargin: Variable input arguments.\n :returns: varargout\n" + == "CLASSMETHOD A function within a package\n\n:param obj: An instance of this class.\n" + ":param varargin: Variable input arguments.\n:returns: varargout" ) @@ -274,11 +289,11 @@ def test_function(mod): func = mod.getter("f_example") assert isinstance(func, doc.MatFunction) assert func.getter("__name__") == "f_example" - assert func.retv == ["o1", "o2", "o3"] - assert func.args == ["a1", "a2"] + assert list(func.retv.keys()) == ["o1", "o2", "o3"] + assert list(func.args.keys()) == ["a1", "a2"] assert ( func.docstring - == " a fun function\n\n :param a1: the first input\n :param a2: another input\n :returns: ``[o1, o2, o3]`` some outputs\n" + == "a fun function\n\n:param a1: the first input\n:param a2: another input\n:returns: ``[o1, o2, o3]`` some outputs" ) @@ -289,7 +304,7 @@ def test_function_getter(mod): assert func.getter("__name__") == "f_example" assert ( func.getter("__doc__") - == " a fun function\n\n :param a1: the first input\n :param a2: another input\n :returns: ``[o1, o2, o3]`` some outputs\n" + == "a fun function\n\n:param a1: the first input\n:param a2: another input\n:returns: ``[o1, o2, o3]`` some outputs" ) assert func.getter("__module__") == "test_data" @@ -299,11 +314,11 @@ def test_package_function(mod): func = mod.getter("f_example") assert isinstance(func, doc.MatFunction) assert func.getter("__name__") == "f_example" - assert func.retv == ["o1", "o2", "o3"] - assert func.args == ["a1", "a2"] + assert list(func.retv.keys()) == ["o1", "o2", "o3"] + assert list(func.args.keys()) == ["a1", "a2"] assert ( func.docstring - == " a fun function\n\n :param a1: the first input\n :param a2: another input\n :returns: ``[o1, o2, o3]`` some outputs\n" + == "a fun function\n\n:param a1: the first input\n:param a2: another input\n:returns: ``[o1, o2, o3]`` some outputs" ) @@ -311,13 +326,13 @@ def test_class_with_get_method(mod): the_class = mod.getter("ClassWithGetMethod") assert isinstance(the_class, doc.MatClass) assert the_class.getter("__name__") == "ClassWithGetMethod" - assert the_class.docstring == " Class with a method named get\n" + assert the_class.docstring == "Class with a method named get" the_method = the_class.getter("get") assert isinstance(the_method, doc.MatMethod) assert the_method.getter("__name__") == "get" - assert the_method.retv == ["varargout"] + assert list(the_method.retv.keys()) == ["varargout"] assert the_method.docstring.startswith( - " Gets the numbers 1-n and fills in the outputs with them" + "Gets the numbers 1-n and fills in the outputs with them" ) diff --git a/tests/test_parse_mfile.py b/tests/test_parse_mfile.py index 24b4c6a..bfd3f78 100644 --- a/tests/test_parse_mfile.py +++ b/tests/test_parse_mfile.py @@ -24,15 +24,15 @@ def test_ClassExample(): assert obj.name == "ClassExample" assert ( obj.docstring - == " test class methods\n\n :param a: the input to :class:`ClassExample`\n" + == "test class methods\n\n:param a: the input to :class:`ClassExample`" ) mymethod = obj.methods["mymethod"] assert mymethod.name == "mymethod" - assert mymethod.retv == ["c"] - assert mymethod.args == ["obj", "b"] + assert list(mymethod.retv.keys()) == ["c"] + assert list(mymethod.args.keys()) == ["obj", "b"] assert ( mymethod.docstring - == " a method in :class:`ClassExample`\n\n :param b: an input to :meth:`mymethod`\n" + == "a method in :class:`ClassExample`\n\n:param b: an input to :meth:`mymethod`" ) @@ -40,108 +40,108 @@ def test_comment_after_docstring(): mfile = os.path.join(TESTDATA_SUB, "f_comment_after_docstring.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_comment_after_docstring", "") assert obj.name == "f_comment_after_docstring" - assert obj.retv == ["output"] - assert obj.args == ["input"] - assert obj.docstring == " Tests a function with comments after docstring\n" + assert list(obj.retv.keys()) == ["output"] + assert list(obj.args.keys()) == ["input"] + assert obj.docstring == "Tests a function with comments after docstring" def test_docstring_no_newline(): mfile = os.path.join(TESTDATA_SUB, "f_docstring_no_newline.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_docstring_no_newline", "") assert obj.name == "f_docstring_no_newline" - assert obj.retv == ["y"] - assert obj.args is None - assert obj.docstring == " Test a function without a newline after docstring\n" + assert list(obj.retv.keys()) == ["y"] + assert list(obj.args.keys()) == [] + assert obj.docstring == "Test a function without a newline after docstring" def test_ellipsis_after_equals(): mfile = os.path.join(TESTDATA_SUB, "f_ellipsis_after_equals.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_ellipsis_after_equals", "") assert obj.name == "f_ellipsis_after_equals" - assert obj.retv == ["output"] - assert obj.args == ["arg"] - assert obj.docstring == " Tests a function with ellipsis after equals\n" + assert list(obj.retv.keys()) == ["output"] + assert list(obj.args.keys()) == ["arg"] + assert obj.docstring == "Tests a function with ellipsis after equals" def test_ellipsis_empty_output(): mfile = os.path.join(TESTDATA_SUB, "f_ellipsis_empty_output.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_ellipsis_empty_output", "") assert obj.name == "f_ellipsis_empty_output" - assert obj.retv is None - assert obj.args == ["arg"] - assert obj.docstring == " Tests a function with ellipsis in the output\n" + assert list(obj.retv.keys()) == [] + assert list(obj.args.keys()) == ["arg"] + assert obj.docstring == "Tests a function with ellipsis in the output" def test_ellipsis_in_comment(): mfile = os.path.join(TESTDATA_SUB, "f_ellipsis_in_comment.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_ellipsis_in_comment", "") assert obj.name == "f_ellipsis_in_comment" - assert obj.retv == ["y"] - assert obj.args == ["x"] - assert obj.docstring == " Tests a function with ellipsis in the comment ...\n" + assert list(obj.retv.keys()) == ["y"] + assert list(obj.args.keys()) == ["x"] + assert obj.docstring == "Tests a function with ellipsis in the comment ..." def test_ellipsis_in_output(): mfile = os.path.join(TESTDATA_SUB, "f_ellipsis_in_output.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_ellipsis_in_output", "") assert obj.name == "f_ellipsis_in_output" - assert obj.retv == ["output", "with", "ellipsis"] - assert obj.args == ["arg"] - assert obj.docstring == " Tests a function with ellipsis in the output\n" + assert list(obj.retv.keys()) == ["output", "with", "ellipsis"] + assert list(obj.args.keys()) == ["arg"] + assert obj.docstring == "Tests a function with ellipsis in the output" def test_ellipsis_in_output_multiple(): mfile = os.path.join(TESTDATA_SUB, "f_ellipsis_in_output_multiple.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_ellipsis_in_output_multiple", "") assert obj.name == "f_ellipsis_in_output_multiple" - assert obj.retv == ["output", "with", "ellipsis"] - assert obj.args == ["arg"] - assert obj.docstring == " Tests a function with multiple ellipsis in the output\n" + assert list(obj.retv.keys()) == ["output", "with", "ellipsis"] + assert list(obj.args.keys()) == ["arg"] + assert obj.docstring == "Tests a function with multiple ellipsis in the output" def test_no_docstring(): mfile = os.path.join(TESTDATA_SUB, "f_no_docstring.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_no_docstring", "") assert obj.name == "f_no_docstring" - assert obj.retv == ["y"] - assert obj.args is None - assert obj.docstring == "" + assert list(obj.retv.keys()) == ["y"] + assert list(obj.args.keys()) == [] + assert obj.docstring is None def test_no_output(): mfile = os.path.join(TESTDATA_SUB, "f_no_output.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_no_output", "") assert obj.name == "f_no_output" - assert obj.retv is None - assert obj.args == ["arg"] - assert obj.docstring == " A function with no outputs\n" + assert list(obj.retv.keys()) == [] + assert list(obj.args.keys()) == ["arg"] + assert obj.docstring == "A function with no outputs" def test_no_input_parentheses(): mfile = os.path.join(TESTDATA_SUB, "f_no_input_parentheses.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_no_input_parentheses", "") assert obj.name == "f_no_input_parentheses" - assert obj.retv == ["y"] - assert obj.args is None - assert obj.docstring == " Tests a function without parentheses in input\n" + assert list(obj.retv.keys()) == ["y"] + assert list(obj.args.keys()) == [] + assert obj.docstring == "Tests a function without parentheses in input" def test_no_spaces(): mfile = os.path.join(TESTDATA_SUB, "f_no_spaces.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_no_spaces", "") assert obj.name == "f_no_spaces" - assert obj.retv == ["a", "b", "c"] - assert obj.args == ["x", "y", "z"] - assert obj.docstring == " Tests a function with no spaces in function signature\n" + assert list(obj.retv.keys()) == ["a", "b", "c"] + assert list(obj.args.keys()) == ["x", "y", "z"] + assert obj.docstring == "Tests a function with no spaces in function signature" def test_with_tabs(): mfile = os.path.join(TESTDATA_SUB, "f_with_tabs.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_with_tabs", "") assert obj.name == "f_with_tabs" - assert obj.retv == ["y"] - assert obj.args == ["x"] - assert obj.docstring == " A function with tabs\n" + assert list(obj.retv.keys()) == ["y"] + assert list(obj.args.keys()) == ["x"] + assert obj.docstring == "A function with tabs" def test_ClassWithEndOfLineComment(): @@ -178,15 +178,15 @@ def test_ClassWithFunctionArguments(): assert obj.name == "ClassWithFunctionArguments" assert ( obj.docstring - == " test class methods with function arguments\n\n :param a: the input to :class:`ClassWithFunctionArguments`\n" + == "test class methods with function arguments\n\n:param a: the input to :class:`ClassWithFunctionArguments`" ) mymethod = obj.methods["mymethod"] assert mymethod.name == "mymethod" - assert mymethod.retv == ["c"] - assert mymethod.args == ["obj", "b"] + assert list(mymethod.retv.keys()) == ["c"] + assert list(mymethod.args.keys()) == ["obj", "b"] assert ( mymethod.docstring - == " a method in :class:`ClassWithFunctionArguments`\n\n :param b: an input to :meth:`mymethod`\n" + == "a method in :class:`ClassWithFunctionArguments`\n\n:param b: an input to :meth:`mymethod`" ) @@ -206,7 +206,7 @@ def test_no_input_no_output_no_parentheses(): assert obj.name == "f_no_input_no_output_no_parentheses" assert ( obj.docstring - == " Tests a function without parentheses in input and no return value\n" + == "Tests a function without parentheses in input and no return value" ) @@ -218,26 +218,26 @@ def test_no_input_no_parentheses_no_docstring(): mfile, "f_no_input_no_parentheses_no_docstring", "test_data" ) assert obj.name == "f_no_input_no_parentheses_no_docstring" - assert obj.retv == ["result"] - assert obj.args is None + assert list(obj.retv.keys()) == ["result"] + assert list(obj.args.keys()) == [] def test_ClassWithCommentHeader(): mfile = os.path.join(DIRNAME, "test_data", "ClassWithCommentHeader.m") obj = mat_types.MatObject.parse_mfile(mfile, "ClassWithCommentHeader", "test_data") assert obj.name == "ClassWithCommentHeader" - assert obj.docstring == " A class with a comment header on the top.\n" + assert obj.docstring == "A class with a comment header on the top." method_get_tform = obj.methods["getTransformation"] assert method_get_tform.name == "getTransformation" - assert method_get_tform.retv == ["tform"] - assert method_get_tform.args == ["obj"] + assert list(method_get_tform.retv.keys()) == ["tform"] + assert list(method_get_tform.args.keys()) == ["obj"] def test_with_comment_header(): mfile = os.path.join(DIRNAME, "test_data", "f_with_comment_header.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_with_comment_header", "test_data") assert obj.name == "f_with_comment_header" - assert obj.docstring == " A simple function with a comment header on the top.\n" + assert obj.docstring == "A simple function with a comment header on the top." def test_script_with_comment_header(): @@ -247,16 +247,15 @@ def test_script_with_comment_header(): ) assert ( obj.docstring - == """ This is a Comment Header - Copyright (C) , by - - Some descriptions ... + == """This is a Comment Header +Copyright (C) , by - This header and all further comments above the first command line - of the script will be ignored by the documentation system. +Some descriptions ... - Lisence (GPL, BSD, etc.) +This header and all further comments above the first command line +of the script will be ignored by the documentation system. +Lisence (GPL, BSD, etc.) """ ) @@ -268,16 +267,15 @@ def test_script_with_comment_header_2(): ) assert ( obj.docstring - == """ This is a Comment Header - Copyright (C) , by + == """This is a Comment Header +Copyright (C) , by - Some descriptions ... +Some descriptions ... - This header and all further comments above the first command line - of the script will be ignored by the documentation system. - - Lisence (GPL, BSD, etc.) +This header and all further comments above the first command line +of the script will be ignored by the documentation system. +Lisence (GPL, BSD, etc.) """ ) @@ -289,9 +287,8 @@ def test_script_with_comment_header_3(): ) assert ( obj.docstring - == """ This is a Comment Header with empty lines above - and many line comments. - + == """This is a Comment Header with empty lines above +and many line comments. """ ) @@ -303,9 +300,8 @@ def test_script_with_comment_header_4(): ) assert ( obj.docstring - == """ This is a Comment Header with a single instruction above - and many line comments. - + == """This is a Comment Header with a single instruction above +and many line comments. """ ) @@ -320,26 +316,34 @@ def test_PropTypeOld(): "docstring": None, "attrs": {}, "default": "'none'", - "specs": "@char", - }, + "size": None, + "type": "char", + "validators": None, + }, # 'type': ['char'] "pos": { "docstring": None, "attrs": {}, "default": "zeros(3,1)", - "specs": "@double vector", - }, + "size": (":", "1"), + "type": "double", + "validators": None, + }, # 'type': ['double', 'vector'], "rotm": { "docstring": None, "attrs": {}, "default": "zeros(3,3)", - "specs": "@double matrix", - }, + "size": (":", ":"), + "type": "double", + "validators": None, + }, # 'type': ['double', 'matrix'], "idx": { "docstring": None, "attrs": {}, "default": "0", - "specs": "@uint8 scalar", - }, + "size": ("1", "1"), + "type": "uint8", + "validators": None, + }, # 'type': ['uint8', 'scalar'], } @@ -356,12 +360,12 @@ def test_ClassWithMethodAttributes(): mfile, "ClassWithMethodAttributes", "test_data" ) assert obj.name == "ClassWithMethodAttributes" - assert obj.docstring == " Class with different method attributes\n" + assert obj.docstring == "Class with different method attributes" assert obj.methods["testNormal"].attrs == {} assert obj.methods["testPublic"].attrs == {"Access": "public"} assert obj.methods["testProtected"].attrs == {"Access": "protected"} assert obj.methods["testPrivate1"].attrs == {"Access": "private"} - assert obj.methods["testPrivate2"].attrs == {"Access": "private"} + assert obj.methods["testPrivate2"].attrs == {"Access": "'private'"} assert obj.methods["testHidden"].attrs == {"Hidden": True} assert obj.methods["testStatic"].attrs == {"Static": True} assert obj.methods["testFriend1"].attrs == {"Access": "?OtherClass"} @@ -376,7 +380,7 @@ def test_ClassWithPropertyAttributes(): mfile, "ClassWithPropertyAttributes", "test_data" ) assert obj.name == "ClassWithPropertyAttributes" - assert obj.docstring == " Class with different property attributes\n" + assert obj.docstring == "Class with different property attributes" assert obj.properties["testNormal"]["attrs"] == {} assert obj.properties["testPublic"]["attrs"] == {"Access": "public"} assert obj.properties["testProtected"]["attrs"] == {"Access": "protected"} @@ -406,16 +410,14 @@ def test_ClassWithoutIndent(): mfile = os.path.join(DIRNAME, "test_data", "ClassWithoutIndent.m") obj = mat_types.MatObject.parse_mfile(mfile, "ClassWithoutIndent", "test_data") assert obj.name == "ClassWithoutIndent" - assert ( - obj.docstring == " First line is not indented\n Second line line is indented\n" - ) + assert obj.docstring == "First line is not indented\nSecond line line is indented" def test_f_with_utf8(): mfile = os.path.join(DIRNAME, "test_data", "f_with_utf8.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_with_utf8", "test_data") assert obj.name == "f_with_utf8" - assert obj.docstring == " Cambia ubicación de partículas.\n" + assert obj.docstring == "Cambia ubicación de partículas." def test_file_parsing_encoding_can_be_specified(): @@ -424,14 +426,14 @@ def test_file_parsing_encoding_can_be_specified(): mfile, "f_with_latin_1", "test_data", encoding="latin-1" ) assert obj.name == "f_with_latin_1" - assert obj.docstring == " Analyse de la réponse à un créneau\n" + assert obj.docstring == "Analyse de la réponse à un créneau" def test_file_parsing_with_no_encoding_specified(): mfile = os.path.join(DIRNAME, "test_data", "f_with_latin_1.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_with_latin_1", "test_data") assert obj.name == "f_with_latin_1" - assert obj.docstring == " Analyse de la r\ufffdponse \ufffd un cr\ufffdneau\n" + assert obj.docstring == r"Analyse de la r\xe9ponse \xe0 un cr\xe9neau" def test_ClassWithBuiltinOverload(): @@ -440,7 +442,7 @@ def test_ClassWithBuiltinOverload(): mfile, "ClassWithBuiltinOverload", "test_data" ) assert obj.name == "ClassWithBuiltinOverload" - assert obj.docstring == " Class that overloads a builtin\n" + assert obj.docstring == "Class that overloads a builtin" def test_ClassWithBuiltinProperties(): @@ -449,14 +451,14 @@ def test_ClassWithBuiltinProperties(): mfile, "ClassWithBuiltinProperties", "test_data" ) assert obj.name == "ClassWithBuiltinProperties" - assert obj.docstring == " Class with properties that overload a builtin\n" + assert obj.docstring == "Class with properties that overload a builtin" assert set(obj.properties) == set(["omega", "alpha", "gamma", "beta"]) - assert obj.properties["omega"]["docstring"] == " a property" - assert obj.properties["alpha"]["docstring"] == (" a property overloading a builtin") + assert obj.properties["omega"]["docstring"] == "a property" + assert obj.properties["alpha"]["docstring"] == ("a property overloading a builtin") assert obj.properties["gamma"]["docstring"] == ( - " a property overloading a builtin with validation" + "a property overloading a builtin with validation" ) - assert obj.properties["beta"]["docstring"] == (" another overloaded property") + assert obj.properties["beta"]["docstring"] == ("another overloaded property") # Fails when running with other test files. Warnings are already logged. @@ -473,7 +475,7 @@ def test_f_with_name_mismatch(caplog): "sphinx.matlab-domain", WARNING, '[sphinxcontrib-matlabdomain] Unexpected function name: "f_name_with_mismatch".' - ' Expected "f_with_name_mismatch" in module "test_data".', + ' Expected "f_with_name_mismatch"in module "test_data".', ), ] @@ -482,16 +484,16 @@ def test_f_with_dummy_argument(): mfile = os.path.join(DIRNAME, "test_data", "f_with_dummy_argument.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_with_dummy_argument", "") assert obj.name == "f_with_dummy_argument" - assert obj.retv == ["obj"] - assert obj.args == ["~", "name"] - assert obj.docstring == " Could be a callback, where first argument is ignored.\n" + assert list(obj.retv.keys()) == ["obj"] + assert list(obj.args.keys()) == ["~", "name"] + assert obj.docstring == "Could be a callback, where first argument is ignored." def test_f_with_string_ellipsis(): mfile = os.path.join(DIRNAME, "test_data", "f_with_string_ellipsis.m") obj = mat_types.MatObject.parse_mfile(mfile, "f_with_string_ellipsis", "test_data") assert obj.name == "f_with_string_ellipsis" - assert obj.docstring == " A function with a string with ellipsis\n" + assert obj.docstring == "A function with a string with ellipsis" def test_ClassWithFunctionVariable(): @@ -500,7 +502,7 @@ def test_ClassWithFunctionVariable(): mfile, "ClassWithFunctionVariable", "test_data" ) assert obj.name == "ClassWithFunctionVariable" - assert obj.docstring == " This line contains functions!\n" + assert obj.docstring == "This line contains functions!" methods = set(obj.methods.keys()) assert methods == {"ClassWithFunctionVariable", "anotherMethodWithFunctions"} @@ -608,9 +610,8 @@ def test_f_with_function_variable(): mfile, "f_with_function_variable", "test_data" ) assert obj.name == "f_with_function_variable" - assert obj.retv == ["obj"] - assert obj.args == ["the_functions", "~"] - print(obj.docstring) + assert list(obj.retv.keys()) == ["obj"] + assert list(obj.args.keys()) == ["the_functions", "~"] def test_ClassWithGetterSetter(): @@ -621,10 +622,12 @@ def test_ClassWithGetterSetter(): assert list(obj.methods.keys()) == ["ClassWithGetterSetter"] assert obj.properties == { "a": { - "docstring": " A nice property", + "docstring": "A nice property", "attrs": {}, "default": None, - "specs": "", + "size": None, + "type": None, + "validators": None, } } @@ -638,7 +641,14 @@ def test_ClassWithDoubleQuotedString(): assert obj.name == "ClassWithDoubleQuotedString" assert set(obj.methods.keys()) == set(["ClassWithDoubleQuotedString", "method1"]) assert obj.properties == { - "Property1": {"docstring": None, "attrs": {}, "default": None, "specs": ""} + "Property1": { + "docstring": None, + "attrs": {}, + "default": None, + "size": None, + "type": None, + "validators": None, + } } @@ -657,9 +667,9 @@ def test_ClassWithDummyArguments(): assert obj.name == "ClassWithDummyArguments" assert set(obj.methods.keys()) == set(["someMethod1", "someMethod2"]) m1 = obj.methods["someMethod1"] - assert m1.args == ["obj", "argument"] + assert list(m1.args.keys()) == ["obj", "argument"] m2 = obj.methods["someMethod2"] - assert m2.args == ["~", "argument"] + assert list(m2.args.keys()) == ["~", "argument"] def test_ClassFolderClassdef(): @@ -669,9 +679,9 @@ def test_ClassFolderClassdef(): assert obj.name == "ClassFolder" assert set(obj.methods.keys()) == set(["ClassFolder", "method_inside_classdef"]) m1 = obj.methods["ClassFolder"] - assert m1.args == ["p"] + assert list(m1.args.keys()) == ["p"] m2 = obj.methods["method_inside_classdef"] - assert m2.args == ["obj", "a", "b"] + assert list(m2.args.keys()) == ["obj", "a", "b"] def test_ClassWithMethodsWithSpaces(): @@ -683,8 +693,7 @@ def test_ClassWithMethodsWithSpaces(): assert obj.name == "ClassWithMethodsWithSpaces" assert set(obj.methods.keys()) == set(["static_method"]) assert ( - obj.docstring - == " Class with methods that have space after the function name.\n" + obj.docstring == "Class with methods that have space after the function name." ) assert obj.methods["static_method"].attrs == {"Static": True} @@ -695,7 +704,7 @@ def test_ClassContainingParfor(): assert isinstance(obj, mat_types.MatClass) assert obj.name == "ClassContainingParfor" assert set(obj.methods.keys()) == set(["test"]) - assert obj.docstring == " Parfor is a keyword\n" + assert obj.docstring == "Parfor is a keyword" def test_ClassWithStringEllipsis(): @@ -704,7 +713,7 @@ def test_ClassWithStringEllipsis(): assert isinstance(obj, mat_types.MatClass) assert obj.name == "ClassWithStringEllipsis" assert set(obj.methods.keys()) == set(["test"]) - assert obj.docstring == " Contains ellipsis in string\n" + assert obj.docstring == "Contains ellipsis in string" def test_ClassLongProperty(): @@ -712,13 +721,13 @@ def test_ClassLongProperty(): obj = mat_types.MatObject.parse_mfile(mfile, "ClassLongProperty", "test_data") assert obj.name == "ClassLongProperty" assert ( - obj.docstring == " test class property with long docstring\n\n " - ":param a: the input to :class:`ClassExample`\n" + obj.docstring == "test class property with long docstring\n\n" + ":param a: the input to :class:`ClassExample`" ) - assert obj.properties["a"]["docstring"] == " short description" + assert obj.properties["a"]["docstring"] == "short description" assert ( - obj.properties["b"]["docstring"] == " A property with a long " - "documentation\n This is the second line\n And a third\n" + obj.properties["b"]["docstring"] == "A property with a long " + "documentation\nThis is the second line\nAnd a third" ) assert obj.properties["c"]["docstring"] is None @@ -730,10 +739,10 @@ def test_ClassWithLongPropertyDocstrings(): ) assert obj.name == "ClassWithLongPropertyDocstrings" assert ( - obj.properties["a"]["docstring"] == " This line is deleted\n" - " This line documents another property\n" + obj.properties["a"]["docstring"] == "This line is deleted\n" + "This line documents another property" ) - assert obj.properties["b"]["docstring"] == " Document this property\n" + assert obj.properties["b"]["docstring"] == "Document this property" def test_ClassWithLongPropertyTrailingEmptyDocstrings(): @@ -745,10 +754,10 @@ def test_ClassWithLongPropertyTrailingEmptyDocstrings(): ) assert obj.name == "ClassWithLongPropertyTrailingEmptyDocstrings" assert ( - obj.properties["a"]["docstring"] == " This line is deleted\n" - " This line documents another property\n" + obj.properties["a"]["docstring"] == "This line is deleted\n" + "This line documents another property" ) - assert obj.properties["b"]["docstring"] == " Document this property\n" + assert obj.properties["b"]["docstring"] == "Document this property" def test_ClassWithPropertyValidators(): @@ -757,10 +766,10 @@ def test_ClassWithPropertyValidators(): mfile, "ClassWithPropertyValidators", "test_data" ) assert obj.name == "ClassWithPropertyValidators" - assert obj.properties["Location"]["docstring"] == " The location\n" - assert obj.properties["Label"]["docstring"] == " The label\n" - assert obj.properties["State"]["docstring"] == " The state\n" - assert obj.properties["ReportLevel"]["docstring"] == " The report level\n" + assert obj.properties["Location"]["docstring"] == "The location" + assert obj.properties["Label"]["docstring"] == "The label" + assert obj.properties["State"]["docstring"] == "The state" + assert obj.properties["ReportLevel"]["docstring"] == "The report level" def test_ClassWithTrailingCommentAfterBases(): @@ -772,46 +781,47 @@ def test_ClassWithTrailingCommentAfterBases(): assert obj.bases == ["handle", "my.super.Class"] assert ( obj.docstring - == " test class methods\n\n :param a: the input to :class:`ClassWithTrailingCommentAfterBases`\n" + == "test class methods\n\n:param a: the input to :class:`ClassWithTrailingCommentAfterBases`" ) mymethod = obj.methods["mymethod"] assert mymethod.name == "mymethod" - assert mymethod.retv == ["c"] - assert mymethod.args == ["obj", "b"] + assert list(mymethod.retv.keys()) == ["c"] + assert list(mymethod.args.keys()) == ["obj", "b"] assert ( mymethod.docstring - == " a method in :class:`ClassWithTrailingCommentAfterBases`\n\n :param b: an input to :meth:`mymethod`\n" + == "a method in :class:`ClassWithTrailingCommentAfterBases`\n\n:param b: an input to :meth:`mymethod`" ) def test_ClassWithEllipsisProperties(): + # TODO change this when the functionality to "nicely" generate one line defaults exists mfile = os.path.join(TESTDATA_ROOT, "ClassWithEllipsisProperties.m") obj = mat_types.MatObject.parse_mfile( mfile, "ClassWithEllipsisProperties", "test_data" ) assert obj.name == "ClassWithEllipsisProperties" assert obj.bases == ["handle"] - assert obj.docstring == " stuff\n" + assert obj.docstring == "stuff" assert len(obj.methods) == 0 - assert obj.properties["A"]["docstring"] == " an expression with ellipsis" - assert obj.properties["A"]["default"] == "1+2+3+4+5" + assert obj.properties["A"]["docstring"] == "an expression with ellipsis" + assert obj.properties["A"]["default"] == "1 + 2 + 3 + 4 + 5" assert ( obj.properties["B"]["docstring"] - == " a cell array with ellipsis and other array notation" + == "a cell array with ellipsis and other array notation" ) - assert obj.properties["B"]["default"].startswith("{'hello','bye';") + assert obj.properties["B"]["default"].startswith("{'hello', 'bye';") assert obj.properties["B"]["default"].endswith("}") - assert obj.properties["C"]["docstring"] == " using end inside array" - assert obj.properties["C"]["default"] == "ClassWithEllipsisProperties.B(2:end,1)" - assert obj.properties["D"]["docstring"] == " String with line continuation" + assert obj.properties["C"]["docstring"] == "using end inside array" + assert obj.properties["C"]["default"] == "ClassWithEllipsisProperties.B(2:end, 1)" + assert obj.properties["D"]["docstring"] == "String with line continuation" assert obj.properties["D"]["default"] == "'...'" - assert obj.properties["E"]["docstring"] == " The string with spaces" + assert obj.properties["E"]["docstring"] == "The string with spaces" assert obj.properties["E"]["default"] == "'some string with spaces'" # mymethod.docstring -# == " a method in :class:`ClassWithTrailingCommentAfterBases`\n\n :param b: an input to :meth:`mymethod`\n" +# == " a method in :class:`ClassWithTrailingCommentAfterBases`\n\n :param b: an input to :meth:`mymethod`" # ) @@ -833,7 +843,7 @@ def test_ClassWithTrailingSemicolons(): ) assert ( obj.docstring - == " Smoothing like it is performed withing Cxx >v7.0 (until v8.2 at least).\n Uses constant 228p_12k frequency vector:\n" + == "Smoothing like it is performed withing Cxx >v7.0 (until v8.2 at least).\nUses constant 228p_12k frequency vector:" ) assert obj.bases == ["hgsetget"] assert list(obj.methods.keys()) == [ @@ -863,7 +873,7 @@ def test_ClassWithSeperatedComments(): assert obj.bases == [] assert "prop" in obj.properties prop = obj.properties["prop"] - assert prop["docstring"] == " Another comment\n" + assert prop["docstring"] == "Another comment" def test_ClassWithKeywordsAsFieldnames(): @@ -878,7 +888,7 @@ def test_ClassWithKeywordsAsFieldnames(): assert "c" in obj.properties assert "calculate" in obj.methods meth = obj.methods["calculate"] - assert meth.docstring == " Returns the value of `d`\n" + assert meth.docstring == "Returns the value of `d`" def test_ClassWithNamedAsArguments(): @@ -888,9 +898,9 @@ def test_ClassWithNamedAsArguments(): assert obj.bases == ["handle", "matlab.mixin.Copyable"] assert "value" in obj.properties meth = obj.methods["arguments"] - assert meth.docstring == " Constructor for arguments\n" + assert meth.docstring == "Constructor for arguments" meth = obj.methods["add"] - assert meth.docstring == " Add new argument\n" + assert meth.docstring == "Add new argument" def test_ClassWithPropertyCellValues(): @@ -914,5 +924,48 @@ def test_ClassWithTests(): assert testRunning.attrs["TestTags"] == ["'Unit'"] +def test_f_with_input_argument_block(): + mfile = os.path.join(DIRNAME, "test_data", "f_with_input_argument_block.m") + obj = mat_types.MatObject.parse_mfile( + mfile, "f_with_input_argument_block", "test_data" + ) + assert obj.name == "f_with_input_argument_block" + assert list(obj.retv.keys()) == ["o1", "o2", "o3"] + assert list(obj.args.keys()) == ["a1", "a2"] + + assert obj.args["a1"]["size"] == ("1", "1") + assert obj.args["a1"]["default"] == "0" + assert obj.args["a1"]["type"] == "double" + assert obj.args["a1"]["docstring"] == "the first input" + + assert obj.args["a2"]["size"] == ("1", "1") + assert obj.args["a2"]["default"] == "a1" + assert obj.args["a1"]["type"] == "double" + assert obj.args["a2"]["docstring"] == "another input" + + +def test_f_with_output_argument_block(): + mfile = os.path.join(DIRNAME, "test_data", "f_with_output_argument_block.m") + obj = mat_types.MatObject.parse_mfile( + mfile, "f_with_output_argument_block", "test_data" + ) + assert obj.name == "f_with_output_argument_block" + assert list(obj.retv.keys()) == ["o1", "o2", "o3"] + assert list(obj.args.keys()) == ["a1", "a2"] + + assert obj.retv["o1"]["size"] == ("1", "1") + assert obj.retv["o1"]["type"] == "double" + assert obj.retv["o1"]["docstring"] == "Output one" + + assert obj.retv["o2"]["size"] == ("1", ":") + assert obj.retv["o2"]["type"] == "double" + assert obj.retv["o2"]["docstring"] == "Another output" + + assert obj.retv["o3"]["size"] == ("1", "1") + assert obj.retv["o3"]["type"] == "double" + assert obj.retv["o3"]["docstring"] == "A third output" + assert obj.retv["o3"]["validators"] == ["mustBePositive"] + + if __name__ == "__main__": pytest.main([os.path.abspath(__file__)]) diff --git a/tox.ini b/tox.ini index 9744f88..deb1103 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311}-sphinx{45,53,60,latest}-pygments{213,latest} +envlist = py{38,39,310,311}-sphinx{45,53,60,latest} [testenv] @@ -18,8 +18,6 @@ deps = sphinx70: Sphinx>=7.0,<8.0 sphinxlatest: Sphinx sphinxdev: https://github.com/sphinx-doc/sphinx/archive/refs/heads/master.zip - pygments213: Pygments>=2.0.1,<2.14.0 - pygmentlatest: Pygments commands = pytest -vv {posargs} tests/ sphinx-build -b html -d {envtmpdir}/doctrees tests/test_docs {envtmpdir}/html