Improved interface for discovery of and access to installed Webviz plugins

tests/test_plugin_init.py

@@ -2,7 +2,9 @@
 
 import mock
 
-from webviz_config.plugins._utils import load_webviz_plugins_with_metadata
+from webviz_config.plugin_registry._load_all_installed_webviz_plugins import (
+    _discover_all_installed_webviz_plugin_entry_points,
+)
 
 
 class DistMock:
@@ -28,33 +30,33 @@ def __init__(self, entry_points, name):
 
 
 def test_no_warning():
-    globals_mock = {}
     with warnings.catch_warnings(record=True) as warn:
-        metadata = load_webviz_plugins_with_metadata(
-            [dist_mock1, dist_mock3], globals_mock
+        discovered_plugins = _discover_all_installed_webviz_plugin_entry_points(
+            [dist_mock1, dist_mock3]
         )
         assert len(warn) == 0, "Too many warnings"
 
-    assert len(metadata) == 2, "Wrong number of items in metadata"
-    assert "SomePlugin1" in globals_mock
-    assert "SomePlugin2" in globals_mock
+    assert len(discovered_plugins) == 2, "Wrong number of items in discovered_plugins"
+    assert "SomePlugin1" in discovered_plugins
+    assert "SomePlugin2" in discovered_plugins
 
 
 def test_warning_multiple():
-    globals_mock = {}
     with warnings.catch_warnings(record=True) as warn:
-        metadata = load_webviz_plugins_with_metadata(
-            [dist_mock1, dist_mock2], globals_mock
+        discovered_plugins = _discover_all_installed_webviz_plugin_entry_points(
+            [dist_mock1, dist_mock2]
         )
 
         assert len(warn) == 1
         assert issubclass(warn[-1].category, RuntimeWarning)
         assert str(warn[-1].message) == (
             "Multiple versions of plugin with name SomePlugin1. "
-            "Already loaded from project dist_mock1. "
-            "Overwriting using plugin with from project dist_mock2"
+            "Already found in project dist_mock1. "
+            "Overwriting using plugin from project dist_mock2"
         )
 
-    assert len(metadata) == 1, "Wrong number of items in metadata"
-    assert metadata["SomePlugin1"]["dist_name"] == "dist_mock2", "Wrong dist name"
-    assert "SomePlugin1" in globals_mock
+    assert len(discovered_plugins) == 1, "Wrong number of items in discovered_plugins"
+    assert "SomePlugin1" in discovered_plugins
+    assert (
+        discovered_plugins["SomePlugin1"].dist_info["dist_name"] == "dist_mock2"
+    ), "Wrong dist name"

tests/test_plugin_metadata.py

@@ -1,9 +1,9 @@
 import webviz_config
-from webviz_config.plugins import metadata
+from webviz_config import plugin_registry
 
 
 def test_webviz_config_metadata():
-    meta = metadata["BannerImage"]
+    meta = plugin_registry.plugin_metadata("BannerImage")
 
     assert meta["dist_name"] == "webviz-config"
     assert meta["dist_version"] == webviz_config.__version__

webviz_config/_config_parser.py

@@ -6,23 +6,36 @@
 
 import yaml
 
-from . import plugins as standard_plugins
+from . import plugin_registry
 from .utils import terminal_colors
-from .utils._get_webviz_plugins import _get_webviz_plugins
 
-SPECIAL_ARGS = ["self", "app", "webviz_settings", "_call_signature"]
+SPECIAL_ARGS = ["self", "app", "webviz_settings", "_plugin_creation_spec"]
+
+
+# Candidate for dataclass once we're past python 3.6
+# pylint: disable=too-few-public-methods
+class PluginCreationSpec:
+    """Contains spec for creating an instance of a webviz plugin"""
+
+    def __init__(
+        self, plugin_name: str, init_arg_str: str, layout_call_signature_str: str
+    ) -> None:
+        self.plugin_name = plugin_name
+        self.init_arg_str = init_arg_str
+        self.layout_call_signature_str = layout_call_signature_str
 
 
 def _call_signature(
     plugin_name: str,
     kwargs: dict,
     config_folder: pathlib.Path,
     contact_person: typing.Optional[dict] = None,
-) -> tuple:
+) -> PluginCreationSpec:
     # pylint: disable=too-many-branches,too-many-statements
     """Takes as input the name of a plugin together with user given arguments
-    (originating from the configuration file). Returns the equivalent Python code wrt.
-    initiating an instance of that plugin (with the given arguments).
+    (originating from the configuration file). Returns a plugin creation spec that
+    has sufficien information to instantiate the wanted plugin instance (with the
+    given arguments).
 
     Raises ParserError in the following scenarios:
       * User is missing a required (i.e. no default value) __init__ argument
@@ -31,7 +44,7 @@ def _call_signature(
       * If there is type mismatch between user given argument value, and type
         hint in __init__ signature (given that type hint exist)
     """
-    argspec = inspect.getfullargspec(getattr(standard_plugins, plugin_name).__init__)
+    argspec = inspect.getfullargspec(plugin_registry.plugin_class(plugin_name).__init__)
 
     if argspec.defaults is not None:
         required_args = argspec.args[: -len(argspec.defaults)]
@@ -116,9 +129,10 @@ def _call_signature(
     if "webviz_settings" in argspec.args:
         special_args += "webviz_settings=webviz_settings, "
 
-    return (
-        f"{plugin_name}({special_args}**{kwargs})",
-        f"plugin_layout(contact_person={contact_person})",
+    return PluginCreationSpec(
+        plugin_name=f"{plugin_name}",
+        init_arg_str=f"{special_args}**{kwargs}",
+        layout_call_signature_str=f"plugin_layout(contact_person={contact_person})",
     )
 
 
@@ -127,9 +141,6 @@ class ParserError(Exception):
 
 
 class ConfigParser:
-
-    STANDARD_PLUGINS = [name for (name, _) in _get_webviz_plugins(standard_plugins)]
-
     def __init__(self, yaml_file: pathlib.Path):
 
         ConfigParser.check_for_tabs_in_file(yaml_file)
@@ -155,6 +166,7 @@ def __init__(self, yaml_file: pathlib.Path):
         self._config_folder = pathlib.Path(yaml_file).parent
         self._page_ids: typing.List[str] = []
         self._assets: set = set()
+        # TODO(Sigurd) Seems to be unused
         self._used_plugin_packages: set = set()
         self.clean_configuration()
 
@@ -260,14 +272,16 @@ def clean_configuration(self) -> None:
                     f"{terminal_colors.END}"
                 )
 
-            plugins = [e for e in page["content"] if isinstance(e, dict)]
+            # Plugin invocations, not direct strings
+            # plugin_invocations
+            plugin_entries_on_page = [e for e in page["content"] if isinstance(e, dict)]
 
-            for plugin in plugins:
-                plugin_name = next(iter(plugin))
-                plugin_variables = next(iter(plugin.values()))
+            for entry in plugin_entries_on_page:
+                plugin_name = next(iter(entry))
+                plugin_variables = next(iter(entry.values()))
                 kwargs = {} if plugin_variables is None else {**plugin_variables}
 
-                if plugin_name not in ConfigParser.STANDARD_PLUGINS:
+                if not plugin_registry.has_plugin(plugin_name):
                     raise ParserError(
                         f"{terminal_colors.RED}{terminal_colors.BOLD}"
                         "You have included a plugin with "
@@ -277,13 +291,15 @@ def clean_configuration(self) -> None:
                         f"{terminal_colors.END}"
                     )
 
-                plugin["_call_signature"] = _call_signature(
+                plugin_creation_spec: PluginCreationSpec = _call_signature(
                     plugin_name,
                     kwargs,
                     self._config_folder,
                 )
+                # Attach the necessary spec for creating plugin instance to the page contents entry
+                entry["_plugin_creation_spec"] = plugin_creation_spec
 
-                self.assets.update(getattr(standard_plugins, plugin_name).ASSETS)
+                self.assets.update(plugin_registry.plugin_class(plugin_name).ASSETS)
 
     @property
     def configuration(self) -> dict:

webviz_config/_docs/_build_docs.py

@@ -14,23 +14,21 @@
 import shutil
 import inspect
 import pathlib
+import sys
 from importlib import import_module
 from collections import defaultdict
 from typing import Any, Dict, Optional, Tuple, List
 
-try:
-    # Python 3.8+
-    # pylint: disable=ungrouped-imports
-    from typing import TypedDict  # type: ignore
-except (ImportError, ModuleNotFoundError):
-    # Python < 3.8
-    from typing_extensions import TypedDict  # type: ignore
+# pylint: disable=wrong-import-position
+if sys.version_info >= (3, 8):
+    from typing import TypedDict  # pylint: disable=no-name-in-module
+else:
+    from typing_extensions import TypedDict  # pylint: disable=no-name-in-module
 
 import jinja2
 
-import webviz_config.plugins
+from .. import plugin_registry
 from .._config_parser import SPECIAL_ARGS
-from ..utils._get_webviz_plugins import _get_webviz_plugins
 
 
 class ArgInfo(TypedDict, total=False):
@@ -72,8 +70,8 @@ def _document_plugin(plugin: Tuple[str, Any]) -> PluginInfo:
         "description": docstring_parts[0] if docstring != "" else None,
         "name": name,
         "package_doc": import_module(subpackage).__doc__,  # type: ignore
-        "dist_name": webviz_config.plugins.metadata[name]["dist_name"],
-        "dist_version": webviz_config.plugins.metadata[name]["dist_version"],
+        "dist_name": plugin_registry.plugin_metadata(name)["dist_name"],
+        "dist_version": plugin_registry.plugin_metadata(name)["dist_version"],
     }
 
     if argspec.defaults is not None:
@@ -105,9 +103,9 @@ def get_plugin_documentation() -> defaultdict:
     """
 
     plugin_doc = [
-        _document_plugin(plugin)
-        for plugin in _get_webviz_plugins(webviz_config.plugins)
-        if not plugin[0].startswith("Example")
+        _document_plugin((plugin_name, plugin_registry.plugin_class(plugin_name)))
+        for plugin_name in plugin_registry.all_plugin_names()
+        if not plugin_name.startswith("Example")
     ]
 
     # Sort the plugins by package:

webviz_config/_plugin_abc.py

@@ -9,7 +9,7 @@
 
 # pylint: disable=wrong-import-position
 if sys.version_info >= (3, 8):
-    from typing import TypedDict
+    from typing import TypedDict  # pylint: disable=no-name-in-module
 else:
     from typing_extensions import TypedDict
 

webviz_config/plugin_registry/__init__.py

@@ -0,0 +1,43 @@
+"""Implements registry that contains all installed plugins
+"""
+
+from typing import Dict, List, Type, Callable
+
+from .._plugin_abc import WebvizPluginABC
+from ._load_all_installed_webviz_plugins import load_all_installed_webviz_plugins
+from ._load_all_installed_webviz_plugins import PluginDistInfo
+from ._load_all_installed_webviz_plugins import LoadedPlugin
+
+try:
+    # Python 3.8+
+    from importlib.metadata import distributions  # type: ignore
+except ModuleNotFoundError:
+    # Python < 3.8
+    from importlib_metadata import distributions  # type: ignore
+
+# Currently do all the imports here during module execution
+# May be refactored to do lazy loading when first accessed through one of the functions below
+_all_loaded_plugins: Dict[str, LoadedPlugin] = load_all_installed_webviz_plugins(
+    distributions()
+)
+
+
+def has_plugin(plugin_name: str) -> bool:
+    return plugin_name in _all_loaded_plugins
+
+
+def plugin_class(plugin_name: str) -> Type[WebvizPluginABC]:
+    return _all_loaded_plugins[plugin_name].plugin_class
+
+
+def plugin_metadata(plugin_name: str) -> PluginDistInfo:
+    return _all_loaded_plugins[plugin_name].dist_info
+
+
+# Something along these lines will probably be needed for validation using pydantic
+def overloaded_init_methods_for_plugin(_plugin_name: str) -> List[Callable]:
+    return []
+
+
+def all_plugin_names() -> List[str]:
+    return list(_all_loaded_plugins)