Merge branch 'master' into pre-commit-ci-update-config

docs/api.rst

@@ -6,17 +6,15 @@ Python API
 **Sphinx-Needs** provides an open API for other Sphinx-extensions to provide specific need-types, create needs or
 make usage of the filter possibilities.
 
-The API allows the injection of extra configuration into it. The API does not support the overall manipulation (e.g remove need types)
+The API allows the injection of extra configuration, but
+does not support manipulation of it (e.g remove need types),
 to keep the final configuration transparent for the Sphinx project authors.
 
-For some implementation ideas, take a look into the Sphinx extension
-`Sphinx-Test-Reports <https://sphinx-test-reports.readthedocs.io/en/latest/>`_ and its
-`source code <https://github.com/useblocks/sphinx-test-reports/blob/master/sphinxcontrib/test_reports/test_reports.py>`_.
-
 .. _api_configuration:
 
 Configuration
 -------------
+
 .. automodule:: sphinx_needs.api.configuration
    :members:
 
@@ -25,17 +23,30 @@ Configuration
 
 Need
 ----
+
 .. automodule:: sphinx_needs.api.need
    :members:
 
 
 Exceptions
 ----------
+
 .. automodule:: sphinx_needs.api.exceptions
    :members:
 
 Data
 ----
 
 .. automodule:: sphinx_needs.data
-   :members: NeedsInfoType, NeedsView, NeedsPartsView
+   :members: NeedsInfoType, NeedsMutable
+
+Views
+-----
+
+These views are returned by certain functions, and injected into filters,
+but should not be instantiated directly.
+
+.. automodule:: sphinx_needs.views
+   :members:
+   :undoc-members:
+   :special-members: __iter__, __getitem__, __len__

docs/conf.py

@@ -47,8 +47,8 @@
     ("py:class", "docutils.nodes.Node"),
     ("py:class", "docutils.parsers.rst.states.RSTState"),
     ("py:class", "docutils.statemachine.StringList"),
-    ("py:class", "T"),
     ("py:class", "sphinx_needs.debug.T"),
+    ("py:class", "sphinx_needs.views._LazyIndexes"),
 ]
 
 rst_epilog = """
@@ -756,7 +756,7 @@ def create_tutorial_needs(app: Sphinx, _env, _docnames):
 
     We do this dynamically, to avoid having to maintain the JSON file manually.
     """
-    all_data = SphinxNeedsData(app.env).get_needs_view()
+    all_data = SphinxNeedsData(app.env).get_needs_mutable()
     writer = NeedsList(app.config, outdir=app.confdir, confdir=app.confdir)
     for i in range(1, 5):
         test_id = f"T_00{i}"

docs/filter.rst

@@ -3,10 +3,8 @@
 Filtering needs
 ===============
 
-**Sphinx-Needs** supports the filtering of need and need_parts by using easy to use options or powerful filter string.
-
-Available options are specific to the used directive, whereas the filter string is supported by all directives and
-roles, which provide filter capabilities.
+The filtering of needs and need parts is supported consistently across numerous directives and roles,
+either by using filter options or by using a filter string.
 
 .. _filter_options:
 
@@ -19,10 +17,8 @@ The following filter options are supported by directives:
  * :ref:`needtable`
  * :ref:`needflow`
  * :ref:`needpie`
- * ``needfilter`` (deprecated!)
  * :ref:`needextend`
 
-
 Related to the used directive and its representation, the filter options create a list of needs, which match the
 filters for status, tags, types and filter.
 
@@ -115,6 +111,7 @@ The usage of a filter string is supported/required by:
 * :ref:`needflow`
 * :ref:`needpie`
 * :ref:`needbar`
+* :ref:`needuml` / :ref:`needarch`
 
 
 The filter string must be a valid Python expression:
@@ -125,11 +122,11 @@ The filter string must be a valid Python expression:
 
 A filter string gets evaluated on needs and need_parts!
 A need_part inherits all options from its parent need, if the need_part has no own content for this option.
-E.g. the need_part *title* is kept, but the *status* attribute is taken from its parent need.
+E.g. the need_part *content* is kept, but the *status* attribute is taken from its parent need.
 
 .. note::
 
-   Following attributes are kept inside a need_part: id, title, links_back
+   The following attributes are kept inside a need_part: id, title, links_back
 
 This allows to perform searches for need_parts, where search options are based on parent attributes.
 
@@ -139,29 +136,7 @@ The following filter will find all need_parts, which are part of a need, which h
 
    :need_count:`is_part and 'car' in tags`
 
-Inside a filter string all the fields of :py:class:`.NeedsInfoType` can be used, including:
-
-* **tags** as Python list (compare like ``"B" in tags``)
-* **type** as Python string (compare like ``"story" == type``)
-* **status** as Python string (compare like ``"opened" != status``)
-* **sections** as Python list with the hierarchy of sections with lowest-level
-  section first.  (compare like ``"Section Header" in sections``)
-* **id** as Python string (compare like ``"MY_ID_" in id``)
-* **title** as Python string (compare like ``len(title.split(" ")) > 5``)
-* **links** as Python list (compare like ``"ID_123" not in links``)
-* **links_back** as Python list (compare like ``"ID_123" not in links_back``)
-* **content** as Python string (compare like ``len(content) == 0``)
-* **is_need** as Python boolean. (compare like ``is_need``)
-* **is_part** as Python boolean. (compare like ``is_part``)
-* **parts** as Python list with :ref:`need_part` of the current need. (compare like ``len(parts)>0``)
-* **sections** as list of sections names, th which the need belongs to.
-* **section_name** as string, which defines the last/lowest section a need belongs to.
-* **docname** as string, which defines the name of the document in which a need is defined, without the extension (similar to Sphinx' ``:doc:`` role)
-* **signature** as string, which contains a function-name, possible set by
-  `sphinx-autodoc <https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html>`_ above the need.
-* **parent_need** as string, which is an id of the need, which has the current need defined in its content
-  (added 0.6.2).
-* **parent_needs** as string, which is a list of need ids (added 0.6.2).
+Inside a filter string all the fields of :py:class:`.NeedsInfoType` can be used, including.
 
 Additional variables for :ref:`need_part`:
 
@@ -172,11 +147,10 @@ Additional variables for :ref:`need_part`:
 .. note:: If extra options were specified using :ref:`needs_extra_options` then
           those will be available for use in filter expressions as well.
 
-
 Finally, the following are available:
 
 * :ref:`re_search`, as Python function for performing searches with a regular expression
-* **needs** as :class:`.NeedsPartsView` object, which contains all needs and need_parts.
+* **needs** as :class:`.NeedsAndPartsListView` object, which contains all needs and need_parts.
 
 If your expression is valid and it's True, the related need is added to the filter result list.
 If it is invalid or returns False, the related need is not taken into account for the current filter.
@@ -214,6 +188,30 @@ If it is invalid or returns False, the related need is not taken into account fo
       .. needfilter::
          :filter: "filter_example" in tags and (("B" in tags or ("spec" == type and "closed" == status)) or "test" == type)
 
+.. _filter_string_performance:
+
+Filter string performance
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 4.0.0
+
+The filter string is evaluated by default for each need and need part
+and, therefore, can be become a performance bottleneck for projects with large numbers of needs.
+
+To improve performance, certain common patterns are identified and optimized by the filter engine, and so using such patterns is recommended:
+
+- ``is_external`` / ``is_external == True`` / ``is_external == False``
+- ``id == 'value'`` / ``id == "value"`` / ``'value' == id`` / ``"value" == id``
+- ``id in ['value1', 'value2', ...]`` / ``id in ("value1", "value2", ...)``
+- ``type == 'value'`` / ``type == "value"`` / ``'value' == type`` / ``"value" == type``
+- ``type in ['value1', 'value2', ...]`` / ``type in ("value1", "value2", ...)``
+- ``status == 'value'`` / ``status == "value"`` / ``'value' == status`` / ``"value" == status``
+- ``status in ['value1', 'value2', ...]`` / ``status in ("value1", "value2", ...)``
+- ``'value' in tags`` / ``"value" in tags``
+
+Also filters containing ``and`` will be split into multiple filters and evaluated separately for the above patterns.
+For example, ``type == 'spec' and other == 'value'`` will first be filtered performantly by ``type == 'spec'`` and then the remaining needs will be filtered by ``other == 'value'``.
+
 .. _re_search:
 
 search
@@ -265,6 +263,8 @@ with the help of Python.
 
 The used code must define a variable ``results``, which must be a list and contains the filtered needs.
 
+The code also has access to a variable called ``needs``, which is a :class:`.NeedsAndPartsListView` instance.
+
 .. need-example::
 
    .. needtable::
@@ -275,18 +275,13 @@ The used code must define a variable ``results``, which must be a list and conta
       # which are linked to each other.
 
       results = []
-      # Lets create a needs_dict to address needs by ids more easily.
-      needs_dict = {x['id']: x for x in needs}
-
-      for need in needs:
-         if need['type'] == 'req':
-            for links_id in need['links']:
-               if needs_dict[links_id]['type'] == 'spec':
-                  results.append(need)
-                  results.append(needs_dict[links_id])
-
-The code has access to a variable called ``needs``, which contains a copy of all needs.
-So manipulations on the values in ``needs`` do not have any affects.
+
+      for need in needs.filter_types(["req"]):
+         for links_id in need['links']:
+            linked_need = needs.get_need(links_id)
+            if linked_need and linked_need['type'] == 'spec':
+               results.append(need)
+               results.append(linked_need)
 
 This mechanism can also be a good alternative for complex filter strings to save performance.
 For example if a filter string is using list comprehensions to get access to linked needs.

docs/tutorial.rst

@@ -11,7 +11,6 @@ We will create need items, link them together, visualize the relationships betwe
     :root_id: T_CAR
     :config: tutorial
     :show_link_names:
-    :show_filters:
     :border_color:
         [status == 'open']:FF0000, 
         [status == 'in progress']:0000FF, 
@@ -22,6 +21,17 @@ We will create need items, link them together, visualize the relationships betwe
     This tutorial assumes that you have already :ref:`installed sphinx-needs <installation>`,
     and that you have a basic understanding of how to use :external+sphinx:doc:`Sphinx <index>` and :external+sphinx:ref:`reStructuredText <rst-primer>`.
 
+Need Lifecycle
+--------------
+
+Within a sphinx build, a primary role of sphinx-needs is to manage the lifecycle of need items:
+
+1. **Collect**: During the read phase, need items are collected from the source files and configured external sources.
+2. **Resolve**: After the read phase, the need items are post-processed to resolve dynamic fields and links, etc, then frozen.
+3. **Analyse**: During the write phase, various directives/roles are available to reference, query, and output analysis of the needs.
+4. **Render**: During the write phase, the need items are rendered into the output format, such as HTML or PDF.
+5. **Validate**: During the final phase, the need items can be validated against configured checks.
+
 Creating need items
 -------------------
 

sphinx_needs/api/__init__.py

@@ -4,7 +4,7 @@
     add_need_type,
     get_need_types,
 )
-from .need import add_external_need, add_need, del_need, make_hashed_id
+from .need import add_external_need, add_need, del_need, get_needs_view, make_hashed_id
 
 __all__ = (
     "add_dynamic_function",
@@ -14,5 +14,6 @@
     "add_need_type",
     "del_need",
     "get_need_types",
+    "get_needs_view",
     "make_hashed_id",
 )

sphinx_needs/api/need.py

@@ -31,6 +31,7 @@
 from sphinx_needs.nodes import Need
 from sphinx_needs.roles.need_part import find_parts, update_need_with_parts
 from sphinx_needs.utils import jinja_parse
+from sphinx_needs.views import NeedsView
 
 logger = get_logger(__name__)
 
@@ -805,3 +806,15 @@ def _merge_global_options(
                 # has at least the key.
                 if key not in needs_info.keys():
                     needs_info[key] = ""
+
+
+def get_needs_view(app: Sphinx) -> NeedsView:
+    """Return a read-only view of all resolved needs.
+
+    .. important:: this should only be called within the write phase,
+        after the needs have been fully collected.
+        If not already done, this will ensure all needs are resolved
+        (e.g. back links have been computed etc),
+        and then lock the data to prevent further modification.
+    """
+    return SphinxNeedsData(app.env).get_needs_view()

sphinx_needs/builder.py

@@ -9,8 +9,6 @@
 
 from sphinx_needs.config import NeedsSphinxConfig
 from sphinx_needs.data import SphinxNeedsData
-from sphinx_needs.directives.need import post_process_needs_data
-from sphinx_needs.filter_common import filter_needs_view
 from sphinx_needs.logging import get_logger, log_warning
 from sphinx_needs.needsfile import NeedsList
 
@@ -58,9 +56,10 @@ def write(
         return super().write(build_docnames, updated_docnames, method)
 
     def finish(self) -> None:
-        post_process_needs_data(self.app)
+        from sphinx_needs.filter_common import filter_needs_view
 
         data = SphinxNeedsData(self.env)
+        needs = data.get_needs_view()
         needs_config = NeedsSphinxConfig(self.env.config)
         filters = data.get_or_create_filters()
         version = getattr(self.env.config, "version", "unset")
@@ -84,7 +83,7 @@ def finish(self) -> None:
 
         filter_string = needs_config.builder_filter
         filtered_needs = filter_needs_view(
-            data.get_needs_view(),
+            needs,
             needs_config,
             filter_string,
             append_warning="(from need_builder_filter)",
@@ -173,7 +172,7 @@ def write(
         pass
 
     def finish(self) -> None:
-        post_process_needs_data(self.app)
+        from sphinx_needs.filter_common import filter_needs_view
 
         data = SphinxNeedsData(self.env)
         version = getattr(self.env.config, "version", "unset")