From d8b0a28beb9084f45d6b6c2e6055e44ccfc59bd5 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 27 Oct 2023 10:49:56 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20Allow=20`needextend`=20to=20use?= =?UTF-8?q?=20dynamic=20functions=20(#1052)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is of note that a key change here is that we alter the needs data post-processing order, to extend the needs, before processing dynamic values, links and constraints. --- .github/workflows/ci.yaml | 2 +- sphinx_needs/directives/need.py | 2 +- sphinx_needs/directives/needextend.py | 76 +++-- tests/__snapshots__/test_needextend.ambr | 298 ++++++++++++++++++ tests/doc_test/doc_needextend_dynamic/conf.py | 13 + .../doc_test/doc_needextend_dynamic/index.rst | 20 ++ tests/test_needextend.py | 14 + 7 files changed, 383 insertions(+), 42 deletions(-) create mode 100644 tests/doc_test/doc_needextend_dynamic/conf.py create mode 100644 tests/doc_test/doc_needextend_dynamic/index.rst diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b7c0b010..21f4648d7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,7 +5,7 @@ jobs: name: "py${{ matrix.python-version }} sp${{ matrix.sphinx-version }} do${{ matrix.docutils-version }} ${{ matrix.os }}" runs-on: ${{ matrix.os }} strategy: - fail-fast: true # Set on "false" to get the results of ALL builds + fail-fast: false # Set on "false" to get the results of ALL builds matrix: os: ["ubuntu-latest"] # 3.9.8 seems to be broken with type_ast diff --git a/sphinx_needs/directives/need.py b/sphinx_needs/directives/need.py index 2636f403a..e21e78742 100644 --- a/sphinx_needs/directives/need.py +++ b/sphinx_needs/directives/need.py @@ -383,12 +383,12 @@ def process_need_nodes(app: Sphinx, doctree: nodes.document, fromdocname: str) - return if not needs_data.needs_is_post_processed: + extend_needs_data(needs, needs_data.get_or_create_extends(), needs_config) resolve_dynamic_values(needs, app) resolve_variants_options(needs, needs_config, app.builder.tags.tags) check_links(needs, needs_config) create_back_links(needs, needs_config) process_constraints(needs, needs_config) - extend_needs_data(needs, needs_data.get_or_create_extends(), needs_config) needs_data.needs_is_post_processed = True for extend_node in doctree.findall(Needextend): diff --git a/sphinx_needs/directives/needextend.py b/sphinx_needs/directives/needextend.py index fdfc3789a..c6a23732f 100644 --- a/sphinx_needs/directives/needextend.py +++ b/sphinx_needs/directives/needextend.py @@ -75,11 +75,7 @@ def extend_needs_data( ) -> None: """Use data gathered from needextend directives to modify fields of existing needs.""" - list_values = ( - ["tags", "links"] - + [x["option"] for x in needs_config.extra_links] - + [f"{x['option']}_back" for x in needs_config.extra_links] - ) # back-links (incoming) + list_values = ["tags", "links"] + [x["option"] for x in needs_config.extra_links] link_names = [x["option"] for x in needs_config.extra_links] for current_needextend in extends.values(): @@ -114,23 +110,26 @@ def extend_needs_data( if option.startswith("+"): option_name = option[1:] if option_name in link_names: - # If we add links, then add all corresponding back links - for ref_need in [i.strip() for i in re.split(";|,", value)]: - if ref_need not in all_needs: - logger.warning( - f"Provided link id {ref_need} for needextend does not exist. [needs]", - type="needs", - location=(current_needextend["docname"], current_needextend["lineno"]), - ) - continue - if ref_need not in need[option_name]: - need[option_name].append(ref_need) - if found_need["id"] not in all_needs[ref_need][f"{option_name}_back"]: - all_needs[ref_need][f"{option_name}_back"] += [found_need["id"]] + if value.strip().startswith("[[") and value.strip().endswith("]]"): # dynamic function + need[option_name].append(value) + else: + for ref_need in [i.strip() for i in re.split(";|,", value)]: + if ref_need not in all_needs: + logger.warning( + f"Provided link id {ref_need} for needextend does not exist. [needs]", + type="needs", + location=(current_needextend["docname"], current_needextend["lineno"]), + ) + continue + if ref_need not in need[option_name]: + need[option_name].append(ref_need) elif option_name in list_values: - for item in [i.strip() for i in re.split(";|,", value)]: - if item not in need[option_name]: - need[option_name].append(item) + if value.strip().startswith("[[") and value.strip().endswith("]]"): # dynamic function + need[option_name].append(value) + else: + for item in [i.strip() for i in re.split(";|,", value)]: + if item not in need[option_name]: + need[option_name].append(item) else: if need[option_name]: # If content is already stored, we need to add some whitespace @@ -140,9 +139,6 @@ def extend_needs_data( elif option.startswith("-"): option_name = option[1:] if option_name in link_names: - # If we remove links, then remove all corresponding back links - for ref_need in (i for i in need[option_name] if i in all_needs): - all_needs[ref_need][f"{option_name}_back"].remove(found_need["id"]) need[option_name] = [] if option_name in list_values: need[option_name] = [] @@ -150,24 +146,24 @@ def extend_needs_data( need[option_name] = "" else: if option in link_names: - # If we change links, then modify all corresponding back links - for ref_need in (i for i in need[option] if i in all_needs): - all_needs[ref_need][f"{option}_back"].remove(found_need["id"]) need[option] = [] - for ref_need in [i.strip() for i in re.split(";|,", value)]: - if ref_need not in all_needs: - logger.warning( - f"Provided link id {ref_need} for needextend does not exist. [needs]", - type="needs", - location=(current_needextend["docname"], current_needextend["lineno"]), - ) - continue - need[option].append(ref_need) - for ref_need in need[option]: - if found_need["id"] not in all_needs[ref_need][f"{option}_back"]: - all_needs[ref_need][f"{option}_back"] += [found_need["id"]] + if value.strip().startswith("[[") and value.strip().endswith("]]"): # dynamic function + need[option].append(value) + else: + for ref_need in [i.strip() for i in re.split(";|,", value)]: + if ref_need not in all_needs: + logger.warning( + f"Provided link id {ref_need} for needextend does not exist. [needs]", + type="needs", + location=(current_needextend["docname"], current_needextend["lineno"]), + ) + continue + need[option].append(ref_need) elif option in list_values: - need[option] = [i.strip() for i in re.split(";|,", value)] + if value.strip().startswith("[[") and value.strip().endswith("]]"): # dynamic function + need[option].append(value) + else: + need[option] = [i.strip() for i in re.split(";|,", value)] else: need[option] = value diff --git a/tests/__snapshots__/test_needextend.ambr b/tests/__snapshots__/test_needextend.ambr index ca3a866b5..b3ace872d 100644 --- a/tests/__snapshots__/test_needextend.ambr +++ b/tests/__snapshots__/test_needextend.ambr @@ -1,4 +1,302 @@ # serializer version: 1 +# name: test_doc_needextend_dynamic[test_app0] + dict({ + 'current_version': '', + 'project': 'Python', + 'versions': dict({ + '': dict({ + 'filters': dict({ + }), + 'filters_amount': 0, + 'needs': dict({ + 'REQ_1': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'REQ_1', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'Requirement 1', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'REQ_1', + 'id_prefix': '', + 'is_external': False, + 'is_modified': True, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + 'REQ_A_1', + 'REQ_B_1', + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 2, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'needextend dynamic functions', + 'sections': list([ + 'needextend dynamic functions', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': None, + 'style': None, + 'tags': list([ + ]), + 'target_id': 'REQ_1', + 'template': None, + 'title': 'Requirement 1', + 'type': 'req', + 'type_name': 'Requirement', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + 'REQ_A_1': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'REQ_A_1', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'Requirement A 1', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'REQ_A_1', + 'id_prefix': '', + 'is_external': False, + 'is_modified': False, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 0, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'needextend dynamic functions', + 'sections': list([ + 'needextend dynamic functions', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': None, + 'style': None, + 'tags': list([ + ]), + 'target_id': 'REQ_A_1', + 'template': None, + 'title': 'Requirement A 1', + 'type': 'req', + 'type_name': 'Requirement', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + 'REQ_B_1': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'REQ_B_1', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'Requirement B 1', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'REQ_B_1', + 'id_prefix': '', + 'is_external': False, + 'is_modified': False, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 0, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'needextend dynamic functions', + 'sections': list([ + 'needextend dynamic functions', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': None, + 'style': None, + 'tags': list([ + ]), + 'target_id': 'REQ_B_1', + 'template': None, + 'title': 'Requirement B 1', + 'type': 'req', + 'type_name': 'Requirement', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + 'REQ_C_1': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'REQ_C_1', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'Requirement C 1', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'REQ_C_1', + 'id_prefix': '', + 'is_external': False, + 'is_modified': False, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 0, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'needextend dynamic functions', + 'sections': list([ + 'needextend dynamic functions', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': None, + 'style': None, + 'tags': list([ + ]), + 'target_id': 'REQ_C_1', + 'template': None, + 'title': 'Requirement C 1', + 'type': 'req', + 'type_name': 'Requirement', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + }), + 'needs_amount': 4, + }), + }), + }) +# --- # name: test_doc_needextend_html[test_app0] dict({ 'current_version': '', diff --git a/tests/doc_test/doc_needextend_dynamic/conf.py b/tests/doc_test/doc_needextend_dynamic/conf.py new file mode 100644 index 000000000..87580e5f4 --- /dev/null +++ b/tests/doc_test/doc_needextend_dynamic/conf.py @@ -0,0 +1,13 @@ +extensions = ["sphinx_needs"] +needs_build_json = True + + +def get_matching_need_ids(app, need, needs, id_prefix=""): + filtered_needs = [] + for need_id, _ in needs.items(): + if id_prefix and need_id.startswith(id_prefix): + filtered_needs.append(need_id) + return filtered_needs + + +needs_functions = [get_matching_need_ids] diff --git a/tests/doc_test/doc_needextend_dynamic/index.rst b/tests/doc_test/doc_needextend_dynamic/index.rst new file mode 100644 index 000000000..17ba8911c --- /dev/null +++ b/tests/doc_test/doc_needextend_dynamic/index.rst @@ -0,0 +1,20 @@ +needextend dynamic functions +============================ + +.. req:: Requirement 1 + :id: REQ_1 + +.. req:: Requirement A 1 + :id: REQ_A_1 + +.. req:: Requirement B 1 + :id: REQ_B_1 + +.. req:: Requirement C 1 + :id: REQ_C_1 + +.. needextend:: REQ_1 + :links: [[get_matching_need_ids("REQ_A_")]] + +.. needextend:: REQ_1 + :+links: [[get_matching_need_ids("REQ_B_")]] diff --git a/tests/test_needextend.py b/tests/test_needextend.py index 0751f87d0..c4095446f 100644 --- a/tests/test_needextend.py +++ b/tests/test_needextend.py @@ -64,3 +64,17 @@ def test_doc_needextend_strict(test_app): "Sphinx error:\nProvided id strict_enable_extend_test for needextend does not exist." in out.stderr.decode("utf-8") ) + + +@pytest.mark.parametrize( + "test_app", [{"buildername": "html", "srcdir": "doc_test/doc_needextend_dynamic"}], indirect=True +) +def test_doc_needextend_dynamic(test_app, snapshot): + app = test_app + app.build() + + # for some reason this intermittently creates incorrect warnings about overriding visitors + # assert app._warning.getvalue() == "" + + needs_data = json.loads(Path(app.outdir, "needs.json").read_text()) + assert needs_data == snapshot(exclude=props("created"))