diff --git a/docs/usage/object.md b/docs/usage/object.md index dcaffcb4..265b210c 100644 --- a/docs/usage/object.md +++ b/docs/usage/object.md @@ -223,19 +223,23 @@ In the current case, the fullname is: Here, -- The first segment `examples` has a link to the top level pakcage `examples`. -- The second segment `styles` has a link to the subpakcage `examples.styles`. -- The third segment `google` has a link to the module `examples.styles.google`. -- The last segment `ExampleClass` is the corresponding object itself so that a link - has been omitted. - -You can check these links by hovering mouse cursor on the name segments. +- The first segment `examples` has a link to the top + level pakcage `examples`. +- The second segment `styles` has a link to the + subpakcage `examples.styles`. +- The third segment `google` has a link to the module + `examples.styles.google`. +- The last segment `ExampleClass` is the corresponding + object itself so that a link has been omitted. + +You can check these links by hovering mouse cursor +on the name segments. ::: examples.styles.google.ExampleClass|sourcelink !!! note - Currently, `__special__` and `_private` members are treated as - a normal member. + Currently, `__special__` and `_private` members + are treated as a normal member. ### Function diff --git a/src/mkapi/importlib.py b/src/mkapi/importlib.py index 51573f9d..ef01b603 100644 --- a/src/mkapi/importlib.py +++ b/src/mkapi/importlib.py @@ -34,11 +34,9 @@ def cache_clear() -> None: """Clear cache. - - mkapi.utils.get_module_node_source, - mkapi.objects.objects - mkapi.importlib.load_module """ - get_module_node_source.cache_clear() load_module.cache_clear() objects.clear() diff --git a/src/mkapi/markdown.py b/src/mkapi/markdown.py index 4c70d032..acc316cc 100644 --- a/src/mkapi/markdown.py +++ b/src/mkapi/markdown.py @@ -25,6 +25,7 @@ def _iter(pattern: re.Pattern, text: str) -> Iterator[re.Match | str]: FENCED_CODE = re.compile(r"^(?P
*[~`]{3,}).*?^(?P=pre)\n?", re.M | re.S) +INLINE_CODE = re.compile(r"(?P`+).+?(?P=pre)") def _iter_fenced_codes(text: str) -> Iterator[re.Match | str]: @@ -235,9 +236,6 @@ def convert(text: str) -> str: return "".join(_convert(text)) -INLINE_CODE = re.compile(r"(?P`+).+?(?P=pre)") - - def finditer(pattern: re.Pattern, text: str) -> Iterator[re.Match | str]: """Yield strings or match objects from a markdown text.""" for match in _iter_fenced_codes(text): diff --git a/src/mkapi/nav.py b/src/mkapi/nav.py index 14865ad2..8d498778 100644 --- a/src/mkapi/nav.py +++ b/src/mkapi/nav.py @@ -12,23 +12,32 @@ from typing import Any -def get_apinav(name: str, predicate: Callable[[str], bool] | None = None) -> list: - """Return list of module names.""" +def split_name_depth(name: str) -> tuple[str, int]: + """Split a nav entry into name and depth.""" if m := re.match(r"^(.+?)\.(\*+)$", name): name, option = m.groups() - n = len(option) - else: - n = 0 + return name, len(option) + return name, 0 + + +def get_apinav(name: str, predicate: Callable[[str], bool] | None = None) -> list: + """Return list of module names.""" + name, depth = split_name_depth(name) + # if m := re.match(r"^(.+?)\.(\*+)$", name): + # name, option = m.groups() + # n = len(option) + # else: + # n = 0 if not get_module_path(name): return [] if not is_package(name): return [name] find = partial(find_submodule_names, predicate=predicate) - if n == 1: + if depth == 1: return [name, *find(name)] - if n == 2: + if depth == 2: return _get_apinav_list(name, find) - if n == 3: + if depth == 3: return [_get_apinav_dict(name, find)] return [name] diff --git a/src/mkapi/plugins.py b/src/mkapi/plugins.py index f6911418..7cf3c03c 100644 --- a/src/mkapi/plugins.py +++ b/src/mkapi/plugins.py @@ -13,10 +13,11 @@ import sys import warnings from pathlib import Path -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING from halo import Halo from mkdocs.config import Config, config_options +from mkdocs.config.defaults import MkDocsConfig from mkdocs.plugins import BasePlugin, get_plugin_logger from mkdocs.structure.files import InclusionLevel, get_files from tqdm.std import tqdm @@ -24,12 +25,15 @@ import mkapi import mkapi.nav from mkapi import renderers +from mkapi.importlib import cache_clear +from mkapi.nav import split_name_depth from mkapi.pages import ( convert_markdown, convert_source, create_object_page, create_source_page, ) +from mkapi.utils import get_module_path, is_module_cache_dirty if TYPE_CHECKING: from collections.abc import Callable @@ -46,35 +50,39 @@ class MkAPIConfig(Config): """Specify the config schema.""" config = config_options.Type(str, default="") - debug = config_options.Type(bool, default=False) - docs_anchor = config_options.Type(str, default="docs") exclude = config_options.Type(list, default=[]) filters = config_options.Type(list, default=[]) - src_anchor = config_options.Type(str, default="source") src_dir = config_options.Type(str, default="src") + docs_anchor = config_options.Type(str, default="docs") + src_anchor = config_options.Type(str, default="source") + debug = config_options.Type(bool, default=False) class MkAPIPlugin(BasePlugin[MkAPIConfig]): """MkAPIPlugin class for API generation.""" - nav: ClassVar[list | None] = None - api_dirs: ClassVar[list] = [] - api_uris: ClassVar[list] = [] - api_srcs: ClassVar[list] = [] - api_uri_width: ClassVar[int] = 0 + api_dirs: list[Path] + api_uris: list[str] + api_srcs: list[str] + + def __init__(self) -> None: + self.api_dirs = [] def on_config(self, config: MkDocsConfig, **kwargs) -> MkDocsConfig: + self.api_uris = [] + self.api_srcs = [] if before_on_config := _get_function("before_on_config", self): before_on_config(config, self) _update_templates(config, self) - _update_config(config, self) + _create_nav(config, self) + _update_nav(config, self) _update_extensions(config, self) if after_on_config := _get_function("after_on_config", self): after_on_config(config, self) return config def on_files(self, files: Files, config: MkDocsConfig, **kwargs) -> Files: - """Collect plugin CSS/JavaScript and append them to `files`.""" + """Collect plugin CSS and append them to `files`.""" for file in files: if file.src_uri.startswith(f"{self.config.src_dir}/"): file.inclusion = InclusionLevel.NOT_IN_NAV @@ -83,11 +91,16 @@ def on_files(self, files: Files, config: MkDocsConfig, **kwargs) -> Files: return files def on_nav(self, *args, **kwargs) -> None: - total = len(MkAPIPlugin.api_uris) + len(MkAPIPlugin.api_srcs) - desc = "MkAPI: Building API pages" - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - self.bar = tqdm(desc=desc, total=total, leave=False) + total = len(self.api_uris) + len(self.api_srcs) + uris = self.api_uris + self.api_srcs + if uris: + self.uri_width = max(len(uri) for uri in uris) + desc = "MkAPI: Building API pages" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.bar = tqdm(desc=desc, total=total, leave=False) + else: + self.bar = None def on_page_markdown(self, markdown: str, page: MkDocsPage, **kwargs) -> str: """Convert Markdown source to intermediate version.""" @@ -112,27 +125,28 @@ def on_page_content( ) -> str: """Merge HTML and MkAPI's object structure.""" toc_title = _get_function("toc_title", self) - if page.file.src_uri in MkAPIPlugin.api_uris: + if page.file.src_uri in self.api_uris: _replace_toc(page.toc, toc_title) self._update_bar(page.file.src_uri) - if page.file.src_uri in MkAPIPlugin.api_srcs: + if page.file.src_uri in self.api_srcs: path = Path(config.docs_dir) / page.file.src_uri html = convert_source(html, path, self.config.docs_anchor) self._update_bar(page.file.src_uri) return html def _update_bar(self, uri: str) -> None: + if not self.bar: + return with warnings.catch_warnings(): warnings.simplefilter("ignore") - uri = uri.ljust(MkAPIPlugin.api_uri_width) + uri = uri.ljust(self.uri_width) self.bar.set_postfix_str(uri, refresh=False) self.bar.update(1) - - def on_post_build(self, *, config: MkDocsConfig) -> None: - self.bar.close() + if self.bar.n == self.bar.total: + self.bar.close() def on_shutdown(self) -> None: - for path in MkAPIPlugin.api_dirs: + for path in self.api_dirs: if path.exists(): logger.info(f"Deleting API directory: {path}") shutil.rmtree(path) @@ -158,30 +172,30 @@ def _update_templates(config: MkDocsConfig, plugin: MkAPIPlugin) -> None: # noq renderers.load_templates() -def _update_config(config: MkDocsConfig, plugin: MkAPIPlugin) -> None: - if not MkAPIPlugin.nav: - _create_nav(config, plugin) - _update_nav(config, plugin) - MkAPIPlugin.nav = config.nav - uris = itertools.chain(MkAPIPlugin.api_uris, MkAPIPlugin.api_srcs) - MkAPIPlugin.api_uri_width = max(len(uri) for uri in uris) - else: - config.nav = MkAPIPlugin.nav - - def _update_extensions(config: MkDocsConfig, plugin: MkAPIPlugin) -> None: # noqa: ARG001 for name in ["admonition", "attr_list", "md_in_html", "pymdownx.superfences"]: if name not in config.markdown_extensions: config.markdown_extensions.append(name) +def _watch_directory(name: str, config: MkDocsConfig) -> None: + if not name: + return + name, depth = split_name_depth(name) + if path := get_module_path(name): + path = str(path.parent if depth else path) + if path not in config.watch: + config.watch.append(path) + + def _create_nav(config: MkDocsConfig, plugin: MkAPIPlugin) -> None: if not config.nav: return - def mkdir(path: str) -> list: + def mkdir(name: str, path: str) -> list: + # _watch_directory(name, config) api_dir = Path(config.docs_dir) / path - if api_dir.exists() and api_dir not in MkAPIPlugin.api_dirs: + if api_dir.exists() and api_dir not in plugin.api_dirs: logger.warning(f"API directory exists: {api_dir}") ans = input("Delete the directory? [yes/no] ") if ans.lower() == "yes": @@ -194,17 +208,17 @@ def mkdir(path: str) -> list: msg = f"Making API directory: {api_dir}" logger.info(msg) api_dir.mkdir() - MkAPIPlugin.api_dirs.append(api_dir) + plugin.api_dirs.append(api_dir) return [] - mkapi.nav.create(config.nav, lambda *args: mkdir(args[1])) - mkdir(plugin.config.src_dir) + mkapi.nav.create(config.nav, lambda *args: mkdir(args[0], args[1])) + mkdir("", plugin.config.src_dir) def _check_path(path: Path): - if path.exists(): - msg = f"Duplicated page: {path.as_posix()!r}" - logger.warning(msg) + # if path.exists(): + # msg = f"Duplicated page: {path.as_posix()!r}" + # logger.warning(msg) if not path.parent.exists(): path.parent.mkdir(parents=True) @@ -217,18 +231,27 @@ def _update_nav(config: MkDocsConfig, plugin: MkAPIPlugin) -> None: section_title = _get_function("section_title", plugin) def _create_page(name: str, path: str, filters: list[str], depth: int) -> str: - n = len(MkAPIPlugin.api_uris) + is_dirty = is_module_cache_dirty(name) + if is_dirty: + cache_clear() + + n = len(plugin.api_uris) spinner.text = f"Collecting modules [{n:>3}]: {name}" - MkAPIPlugin.api_uris.append(path) + abs_path = Path(config.docs_dir) / path - _check_path(abs_path) - create_object_page(f"{name}.**", abs_path, [*filters, "sourcelink"]) + + if not abs_path.exists(): + _check_path(abs_path) + create_object_page(f"{name}.**", abs_path, [*filters, "sourcelink"]) + plugin.api_uris.append(path) path = plugin.config.src_dir + "/" + name.replace(".", "/") + ".md" - MkAPIPlugin.api_srcs.append(path) abs_path = Path(config.docs_dir) / path - _check_path(abs_path) - create_source_page(f"{name}.**", abs_path, filters) + + if not abs_path.exists(): + _check_path(abs_path) + create_source_page(f"{name}.**", abs_path, filters) + plugin.api_srcs.append(path) return page_title(name, depth) if page_title else name diff --git a/src/mkapi/utils.py b/src/mkapi/utils.py index 909c4f81..aedaf892 100644 --- a/src/mkapi/utils.py +++ b/src/mkapi/utils.py @@ -3,6 +3,7 @@ import ast import re +from dataclasses import dataclass from functools import cache from importlib.util import find_spec from pathlib import Path @@ -12,6 +13,7 @@ from collections.abc import Callable, Iterable, Iterator +@cache def get_module_path(name: str) -> Path | None: """Return the source path of the module name.""" try: @@ -71,17 +73,48 @@ def find_submodule_names( return names -@cache +@dataclass +class ModuleCache: + """Cache for module node and source.""" + + name: str + mtime: float + node: ast.Module + source: str + + +module_cache: dict[str, ModuleCache | None] = {} + + +def is_module_cache_dirty(name: str) -> bool: + """Return True if `module_cache` is dirty.""" + if not (path := get_module_path(name)): + return False + if not (cache := module_cache.get(name)): + return True + return cache.mtime != path.stat().st_mtime + + def get_module_node_source(name: str) -> tuple[ast.Module, str] | None: """Return a tuple of ([ast.Module], source) from a module name.""" + if name in module_cache and not module_cache[name]: + return None if not (path := get_module_path(name)): + module_cache[name] = None return None + mtime = path.stat().st_mtime + if (cache := module_cache.get(name)) and cache.mtime == mtime: + return cache.node, cache.source with path.open("r", encoding="utf-8") as f: source = f.read() node = ast.parse(source) + module_cache[name] = ModuleCache(name, mtime, node, source) return node, source +get_module_node_source.cache_clear = lambda: module_cache.clear() + + def get_module_node(name: str) -> ast.Module | None: """Return an [ast.Module] instance from a module name.""" if node_source := get_module_node_source(name): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 2b116127..5fc168b4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -70,7 +70,6 @@ def mkapi_plugin(mkdocs_config: MkDocsConfig): def test_mkapi_plugin(mkapi_plugin: MkAPIPlugin): assert isinstance(mkapi_plugin, MkAPIPlugin) - assert mkapi_plugin.nav is None assert isinstance(mkapi_plugin.config, MkAPIConfig) @@ -122,5 +121,5 @@ def test_on_config(config: MkDocsConfig, mkapi_plugin: MkAPIPlugin): assert (Path(config.docs_dir) / path).exists() -def test_build(config: MkDocsConfig): - assert build(config) is None +# def test_build(config: MkDocsConfig): +# assert build(config) is None diff --git a/tests/test_utils.py b/tests/test_utils.py index c09432b5..11990d2c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,16 @@ +import sys +import time +from pathlib import Path + from mkapi.utils import ( find_submodule_names, get_module_node, + get_module_node_source, get_module_path, + is_module_cache_dirty, is_package, iter_submodule_names, + module_cache, ) @@ -39,3 +46,40 @@ def test_get_module_node(): assert node1 node2 = get_module_node("mkapi") assert node1 is node2 + + +def test_module_cache(tmpdir: Path): + module_cache.clear() + assert not get_module_node_source("___") + assert "___" in module_cache + assert not is_module_cache_dirty("a") + + sys.path.insert(0, str(tmpdir)) + + path = tmpdir / "a.py" + source = "1\n" + with path.open("w") as f: + f.write(source) + get_module_path.cache_clear() + path_ = get_module_path("a") + assert path_ + assert is_module_cache_dirty("a") + x = get_module_node_source("a") + assert x + assert x[1] == source + assert "a" in module_cache + assert not is_module_cache_dirty("a") + y = get_module_node_source("a") + assert y + assert x[0] is y[0] + time.sleep(0.01) + source = "2\n" + with path.open("w") as f: + f.write(source) + assert is_module_cache_dirty("a") + z = get_module_node_source("a") + assert z + assert x[0] is not z[0] + assert z[1] == source + + sys.path.pop(0)