diff --git a/sphinx_needs/api/need.py b/sphinx_needs/api/need.py index 00394fe1c..34c104cfa 100644 --- a/sphinx_needs/api/need.py +++ b/sphinx_needs/api/need.py @@ -49,6 +49,7 @@ def add_need( id: str | None = None, content: str | StringList = "", lineno_content: None | int = None, + doctype: None | str = None, status: str | None = None, tags: None | str | list[str] = None, constraints: None | str | list[str] = None, @@ -290,7 +291,8 @@ def run(): trimmed_title = title[: max_length - 3] + "..." # Calculate doc type, e.g. .rst or .md - doctype = os.path.splitext(env.doc2path(docname))[1] if docname else "" + if doctype is None: + doctype = os.path.splitext(env.doc2path(docname))[1] if docname else "" # Add the need and all needed information needs_info: NeedsInfoType = { diff --git a/sphinx_needs/data.py b/sphinx_needs/data.py index 46d7f0a38..0fb95fc4e 100644 --- a/sphinx_needs/data.py +++ b/sphinx_needs/data.py @@ -63,6 +63,10 @@ class CoreFieldParameters(TypedDict): """JSON schema for the field.""" show_in_layout: NotRequired[bool] """Whether to show the field in the rendered layout of the need by default (False if not present).""" + exclude_external: NotRequired[bool] + """Whether field should be excluded when loading external needs (False if not present).""" + exclude_import: NotRequired[bool] + """Whether field should be excluded when importing needs (False if not present).""" exclude_json: NotRequired[bool] """Whether field should be part of the default exclusions from the JSON representation (False if not present).""" @@ -72,16 +76,22 @@ class CoreFieldParameters(TypedDict): "docname": { "description": "Name of the document where the need is defined (None if external).", "schema": {"type": ["string", "null"], "default": None}, + "exclude_external": True, + "exclude_import": True, }, "lineno": { "description": "Line number where the need is defined (None if external).", "schema": {"type": ["integer", "null"], "default": None}, "exclude_json": True, + "exclude_external": True, + "exclude_import": True, }, "lineno_content": { "description": "Line number on which the need content starts (None if external).", "schema": {"type": ["integer", "null"], "default": None}, "exclude_json": True, + "exclude_external": True, + "exclude_import": True, }, "full_title": { "description": "Title of the need, of unlimited length.", @@ -105,26 +115,32 @@ class CoreFieldParameters(TypedDict): "description": "Hide the meta-data information of the need.", "schema": {"type": "boolean", "default": False}, "exclude_json": True, + "exclude_external": True, }, "hide": { "description": "If true, the need is not rendered.", "schema": {"type": "boolean", "default": False}, "exclude_json": True, + "exclude_external": True, }, "delete": { "description": "If true, the need is deleted entirely.", "schema": {"type": "boolean", "default": False}, "show_in_layout": True, + "exclude_external": True, + "exclude_import": True, }, "layout": { "description": "Key of the layout, which is used to render the need.", "schema": {"type": ["string", "null"], "default": None}, "show_in_layout": True, + "exclude_external": True, }, "style": { "description": "Comma-separated list of CSS classes (all appended by `needs_style_`).", "schema": {"type": ["string", "null"], "default": None}, "show_in_layout": True, + "exclude_external": True, }, "arch": { "description": "Mapping of uml key to uml content.", @@ -133,19 +149,25 @@ class CoreFieldParameters(TypedDict): "additionalProperties": {"type": "string"}, "default": {}, }, + "exclude_external": True, + "exclude_import": True, }, "is_external": { "description": "If true, no node is created and need is referencing external url.", "schema": {"type": "boolean", "default": False}, + "exclude_external": True, + "exclude_import": True, }, "external_url": { "description": "URL of the need, if it is an external need.", "schema": {"type": ["string", "null"], "default": None}, "show_in_layout": True, + "exclude_import": True, }, "external_css": { "description": "CSS class name, added to the external reference.", "schema": {"type": "string", "default": ""}, + "exclude_import": True, }, "type": { "description": "Type of the need.", @@ -154,37 +176,53 @@ class CoreFieldParameters(TypedDict): "type_name": { "description": "Name of the type.", "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, "type_prefix": { "description": "Prefix of the type.", "schema": {"type": "string", "default": ""}, "exclude_json": True, + "exclude_external": True, + "exclude_import": True, }, "type_color": { "description": "Hexadecimal color code of the type.", "schema": {"type": "string", "default": ""}, "exclude_json": True, + "exclude_external": True, + "exclude_import": True, }, "type_style": { "description": "Style of the type.", "schema": {"type": "string", "default": ""}, "exclude_json": True, + "exclude_external": True, + "exclude_import": True, }, "is_modified": { "description": "Whether the need was modified by needextend.", "schema": {"type": "boolean", "default": False}, + "exclude_external": True, + "exclude_import": True, }, "modifications": { "description": "Number of modifications by needextend.", "schema": {"type": "integer", "default": 0}, + "exclude_external": True, + "exclude_import": True, }, "is_need": { "description": "Whether the need is a need.", "schema": {"type": "boolean", "default": True}, + "exclude_external": True, + "exclude_import": True, }, "is_part": { "description": "Whether the need is a part.", "schema": {"type": "boolean", "default": False}, + "exclude_external": True, + "exclude_import": True, }, "parts": { "description": "Mapping of parts, a.k.a. sub-needs, IDs to data that overrides the need's data", @@ -193,36 +231,46 @@ class CoreFieldParameters(TypedDict): "additionalProperties": {"type": "object"}, "default": {}, }, + "exclude_external": True, + "exclude_import": True, }, "id_parent": { "description": ", or if not a part.", "exclude_json": True, "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, "id_complete": { "description": "., or if not a part.", "exclude_json": True, "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, "jinja_content": { "description": "Whether the content should be pre-processed by jinja.", "schema": {"type": "boolean", "default": False}, "show_in_layout": True, + "exclude_external": True, }, "template": { "description": "Template of the need.", "schema": {"type": ["string", "null"], "default": None}, "show_in_layout": True, + "exclude_external": True, }, "pre_template": { "description": "Pre-template of the need.", "schema": {"type": ["string", "null"], "default": None}, "show_in_layout": True, + "exclude_external": True, }, "post_template": { "description": "Post-template of the need.", "schema": {"type": ["string", "null"], "default": None}, "show_in_layout": True, + "exclude_external": True, }, "content": { "description": "Content of the need.", @@ -231,18 +279,26 @@ class CoreFieldParameters(TypedDict): "pre_content": { "description": "Pre-content of the need.", "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, "post_content": { "description": "Post-content of the need.", "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, "has_dead_links": { "description": "True if any links reference need ids that are not found in the need list.", "schema": {"type": "boolean", "default": False}, + "exclude_external": True, + "exclude_import": True, }, "has_forbidden_dead_links": { "description": "True if any links reference need ids that are not found in the need list, and the link type does not allow dead links.", "schema": {"type": "boolean", "default": False}, + "exclude_external": True, + "exclude_import": True, }, "constraints": { "description": "List of constraint names, which are defined for this need.", @@ -255,15 +311,21 @@ class CoreFieldParameters(TypedDict): "additionalProperties": {"type": "object"}, "default": {}, }, + "exclude_external": True, + "exclude_import": True, }, "constraints_passed": { "description": "True if all constraints passed, False if any failed, None if not yet checked.", "schema": {"type": "boolean", "default": True}, + "exclude_external": True, + "exclude_import": True, }, "constraints_error": { "description": "An error message set if any constraint failed, and `error_message` field is set in config.", "schema": {"type": "string", "default": ""}, "show_in_layout": True, + "exclude_external": True, + "exclude_import": True, }, "doctype": { "description": "Type of the document where the need is defined, e.g. '.rst'.", @@ -272,19 +334,25 @@ class CoreFieldParameters(TypedDict): "sections": { "description": "Sections of the need.", "schema": {"type": "array", "items": {"type": "string"}, "default": []}, + "exclude_import": True, }, "section_name": { "description": "Simply the first section.", "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, "signature": { "description": "Derived from a docutils desc_name node.", "schema": {"type": "string", "default": ""}, "show_in_layout": True, + "exclude_import": True, }, "parent_need": { "description": "Simply the first parent id.", "schema": {"type": "string", "default": ""}, + "exclude_external": True, + "exclude_import": True, }, } diff --git a/sphinx_needs/directives/needimport.py b/sphinx_needs/directives/needimport.py index 863b1d463..d216813f5 100644 --- a/sphinx_needs/directives/needimport.py +++ b/sphinx_needs/directives/needimport.py @@ -14,7 +14,7 @@ from sphinx_needs.api import add_need from sphinx_needs.config import NEEDS_CONFIG, NeedsSphinxConfig -from sphinx_needs.data import NeedsInfoType +from sphinx_needs.data import NeedsCoreFields, NeedsInfoType from sphinx_needs.debug import measure_time from sphinx_needs.defaults import string_to_boolean from sphinx_needs.filter_common import filter_single_need @@ -198,6 +198,23 @@ def run(self) -> Sequence[nodes.Node]: import_prefix_link_edit(needs_list, id_prefix, needs_config.extra_links) + # all known need fields in the project + known_keys = { + *NeedsCoreFields, + *(x["option"] for x in needs_config.extra_links), + *(x["option"] + "_back" for x in needs_config.extra_links), + *NEEDS_CONFIG.extra_options, + } + # all keys that should not be imported from external needs + omitted_keys = { + *(k for k, v in NeedsCoreFields.items() if v.get("exclude_import")), + *(x["option"] + "_back" for x in needs_config.extra_links), + } + + # collect unknown keys to log them + unknown_keys: set[str] = set() + + # directive options that can be override need fields override_options = ( "collapse", "style", @@ -206,59 +223,52 @@ def run(self) -> Sequence[nodes.Node]: "pre_template", "post_template", ) - known_options = ( - # need general parameters - "need_type", - "title", - "id", - "content", - "status", - "tags", - "constraints", - # need render parameters - "jinja_content", - "hide", - "collapse", - "style", - "layout", - "template", - "pre_template", - "post_template", - # note we omit locational parameters, such as signature and sections - # since these will be computed again for the new location - *[x["option"] for x in needs_config.extra_links], - *NEEDS_CONFIG.extra_options, - ) + need_nodes = [] for need_params in needs_list.values(): + if "description" in need_params and not need_params.get("content"): + # legacy versions of sphinx-needs changed "description" to "content" when outputting to json + need_params["content"] = need_params["description"] # type: ignore[typeddict-item] + del need_params["description"] # type: ignore[typeddict-item] + + # Remove unknown options, as they may be defined in source system, but not in this sphinx project + for option in list(need_params): + if option not in known_keys: + unknown_keys.add(option) + del need_params[option] # type: ignore[misc] + elif option in omitted_keys: + del need_params[option] # type: ignore[misc] + for override_option in override_options: if override_option in self.options: need_params[override_option] = self.options[override_option] # type: ignore[literal-required] if "hide" in self.options: need_params["hide"] = True - # The key needs to be different for add_need() api call. - need_params["need_type"] = need_params["type"] # type: ignore[typeddict-unknown-key] + # These keys need to be different for add_need() api call. + need_params["need_type"] = need_params.pop("type", "") # type: ignore[misc,typeddict-unknown-key] + need_params["title"] = need_params.pop( + "full_title", need_params.get("title", "") + ) # type: ignore[misc] # Replace id, to get unique ids need_params["id"] = id_prefix + need_params["id"] - if "description" in need_params and not need_params.get("content"): - # legacy versions of sphinx-needs changed "description" to "content" when outputting to json - need_params["content"] = need_params["description"] # type: ignore[typeddict-item] - del need_params["description"] # type: ignore[typeddict-item] - - # Remove unknown options, as they may be defined in source system, but not in this sphinx project - for option in list(need_params): - if option not in known_options: - del need_params[option] # type: ignore - + # override location need_params["docname"] = self.docname need_params["lineno"] = self.lineno nodes = add_need(self.env.app, self.state, **need_params) # type: ignore[call-arg] need_nodes.extend(nodes) + if unknown_keys: + log_warning( + logger, + f"Unknown keys in import need source: {sorted(unknown_keys)!r}", + "unknown_import_keys", + location=self.get_location(), + ) + add_doc(self.env, self.env.docname) return need_nodes diff --git a/sphinx_needs/external_needs.py b/sphinx_needs/external_needs.py index c04afe379..0283b65f9 100644 --- a/sphinx_needs/external_needs.py +++ b/sphinx_needs/external_needs.py @@ -11,8 +11,8 @@ from sphinx.environment import BuildEnvironment from sphinx_needs.api import add_external_need, del_need -from sphinx_needs.config import NeedsSphinxConfig -from sphinx_needs.data import SphinxNeedsData +from sphinx_needs.config import NEEDS_CONFIG, NeedsSphinxConfig +from sphinx_needs.data import NeedsCoreFields, SphinxNeedsData from sphinx_needs.logging import get_logger, log_warning from sphinx_needs.utils import clean_log, import_prefix_link_edit @@ -117,30 +117,43 @@ def load_external_needs(app: Sphinx, env: BuildEnvironment, docname: str) -> Non id_prefix = source.get("id_prefix", "").upper() import_prefix_link_edit(needs, id_prefix, needs_config.extra_links) - known_options = ( - # need general parameters - "need_type", - "title", - "id", - "content", - "status", - "tags", - "constraints", - # need locational parameters - "signature", - "sections", - # note we omit render parameters, such as style and layout - "external_css", - "external_url", - *[x["option"] for x in needs_config.extra_links], - *needs_config.extra_options, # TODO should this be NEEDS_CONFIG.extra_options? - ) + # all known need fields in the project + known_keys = { + *NeedsCoreFields, + *(x["option"] for x in needs_config.extra_links), + *(x["option"] + "_back" for x in needs_config.extra_links), + *NEEDS_CONFIG.extra_options, + } + # all keys that should not be imported from external needs + omitted_keys = { + *(k for k, v in NeedsCoreFields.items() if v.get("exclude_external")), + *(x["option"] + "_back" for x in needs_config.extra_links), + } + + # collect unknown keys to log them + unknown_keys: set[str] = set() for need in needs.values(): need_params = {**defaults, **need} - # The key needs to be different for add_need() api call. - need_params["need_type"] = need["type"] + if "description" in need_params and not need_params.get("content"): + # legacy versions of sphinx-needs changed "description" to "content" when outputting to json + need_params["content"] = need_params["description"] + del need_params["description"] + + # Remove unknown options, as they may be defined in source system, but not in this sphinx project + for option in list(need_params): + if option not in known_keys: + unknown_keys.add(option) + del need_params[option] + elif option in omitted_keys: + del need_params[option] + + # These keys need to be different for add_need() api call. + need_params["need_type"] = need_params.pop("type", "") + need_params["title"] = need_params.pop( + "full_title", need_params.get("title", "") + ) # Replace id, to get unique ids need_params["id"] = id_prefix + need["id"] @@ -157,16 +170,6 @@ def load_external_needs(app: Sphinx, env: BuildEnvironment, docname: str) -> Non f'{source["base_url"]}/{need.get("docname", "__error__")}.html#{need["id"]}' ) - if "description" in need_params and not need_params.get("content"): - # legacy versions of sphinx-needs changed "description" to "content" when outputting to json - need_params["content"] = need_params["description"] - del need_params["description"] - - # Remove unknown options, as they may be defined in source system, but not in this sphinx project - for option in list(need_params): - if option not in known_options: - del need_params[option] - # check if external needs already exist ext_need_id = need_params["id"] @@ -189,6 +192,15 @@ def load_external_needs(app: Sphinx, env: BuildEnvironment, docname: str) -> Non add_external_need(app, **need_params) + if unknown_keys: + location = source.get("json_url", "") or source.get("json_path", "") + log_warning( + log, + f"Unknown keys in external need source: {sorted(unknown_keys)!r}", + "unknown_external_keys", + location=location, + ) + class NeedsExternalException(BaseException): pass diff --git a/tests/__snapshots__/test_external.ambr b/tests/__snapshots__/test_external.ambr index 1d8bbf3e2..3e5c793a3 100644 --- a/tests/__snapshots__/test_external.ambr +++ b/tests/__snapshots__/test_external.ambr @@ -5,7 +5,6 @@ '1.3': dict({ 'needs': dict({ 'EXT_REQ_01': dict({ - 'doctype': '', 'external_css': 'external_link', 'external_url': 'http://my_company.com/docs/v1/index.html#REQ_01', 'full_title': 'REQ_01',