Merge pull request #4 from eccenca/develop

v1.0.0beta2

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/)
 
+## [1.0.0beta2] 2024-07-04
+
+### Fixed
+
+- `prov:wasGeneratedBy` in output graphs now refers to a plugin IRI instead of a literal
+
+### Changed
+
+- keep original output ("No explanations found.") if no inconsistencies found with Validate plugin
+- provenance data in output graphs now includes plugin parameter settings
+- new icons
+
+
 ## [1.0.0beta1] 2024-07-01
 
 ### Fixed
@@ -13,14 +26,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 ### Changed
 
 - complete validation for IRI parameters
-- Remove "Annnotate inferred subclass axioms" parameter
+- remove "Annnotate inferred subclass axioms" parameter in Reason plugin
+- new icons
 
 ## [1.0.0alpha3] 2024-06-28
 
 ### Added
 
 - "Annotate inferred axioms" parameter in Reason plugin
-- "Maximum RAM percentage" parameter in Reason and Validate plugins
+- "Maximum RAM percentage" parameter
 
 ### Changed
 

README.md

@@ -90,7 +90,7 @@ Maximum heap size for the Java virtual machine in the DI container running the r
 
 # Validate
 
-In case ontology inconsistencies are found, the plugin outputs the explanation as text in Markdown format using the path "text".
+The plugin outputs the explanation as text in Markdown format using the path "text".
 
 ## Options
 
@@ -122,6 +122,10 @@ The IRI of the output graph for the reasoning result.
 
 If enabled, an explanation markdown file is written to the project.
 
+### Output filename
+
+The filename of the Markdown file with the explanation of inconsistencies.
+
 :warning: Existing files will be overwritten.
 
 ### Stop at inconsistencies

cmem_plugin_reason/plugin_reason.py

@@ -25,19 +25,19 @@
     ROBOT,
     create_xml_catalog_file,
     get_graphs_tree,
+    post_provenance,
     remove_temp,
     send_result,
 )
 
 
 @Plugin(
     label="Reason",
-    icon=Icon(file_name="obofoundry.png", package=__package__),
-    description="Given a data and an ontology graph, this task performs reasoning using ROBOT.",
-    documentation="""A task performing reasoning using ROBOT (ROBOT is an OBO Tool).
-    It takes an OWL ontology and a data graph as inputs and writes the reasoning result
-    to a specified graph. The following reasoner options are supported: ELK, Expression
-    Materializing Reasoner, HermiT, JFact, Structural Reasoner and Whelk.""",
+    icon=Icon(file_name="reason.png", package=__package__),
+    description="Performs OWL reasoning.",
+    documentation="""A task performing OWL reasoning. With an OWL ontology and a data graph as input
+    the reasoning result is written to a specified graph. The following reasoners are supported:
+    ELK, Expression Materializing Reasoner, HermiT, JFact, Structural Reasoner and Whelk.""",
     parameters=[
         REASONER_PARAMETER,
         ONTOLOGY_GRAPH_IRI_PARAMETER,
@@ -56,7 +56,7 @@
         ),
         PluginParameter(
             param_type=StringParameterType(),
-            name="result_graph_iri",
+            name="output_graph_iri",
             label="Result graph IRI",
             description="The IRI of the output graph for the reasoning result. ⚠️ Existing graphs "
             "will be overwritten.",
@@ -168,7 +168,7 @@ def __init__(  # noqa: PLR0913
         self,
         data_graph_iri: str = "",
         ontology_graph_iri: str = "",
-        result_graph_iri: str = "",
+        output_graph_iri: str = "",
         reasoner: str = "elk",
         class_assertion: bool = False,
         data_property_characteristic: bool = False,
@@ -207,11 +207,11 @@ def __init__(  # noqa: PLR0913
             errors += 'Invalid IRI for parameter "Data graph IRI". '
         if not validators.url(ontology_graph_iri):
             errors += 'Invalid IRI for parameter "Ontology graph IRI". '
-        if not validators.url(result_graph_iri):
+        if not validators.url(output_graph_iri):
             errors += 'Invalid IRI for parameter "Result graph IRI". '
-        if result_graph_iri and result_graph_iri == data_graph_iri:
+        if output_graph_iri and output_graph_iri == data_graph_iri:
             errors += "Result graph IRI cannot be the same as the data graph IRI. "
-        if result_graph_iri and result_graph_iri == ontology_graph_iri:
+        if output_graph_iri and output_graph_iri == ontology_graph_iri:
             errors += "Result graph IRI cannot be the same as the ontology graph IRI. "
         if reasoner not in REASONERS:
             errors += 'Invalid value for parameter "Reasoner". '
@@ -221,9 +221,23 @@ def __init__(  # noqa: PLR0913
             errors += 'Invalid value for parameter "Maximum RAM Percentage". '
         if errors:
             raise ValueError(errors[:-1])
+        self.sub_class = sub_class
+        self.equivalent_class = equivalent_class
+        self.disjoint_classes = disjoint_classes
+        self.data_property_characteristic = data_property_characteristic
+        self.equivalent_data_properties = equivalent_data_properties
+        self.sub_data_property = sub_data_property
+        self.class_assertion = class_assertion
+        self.property_assertion = property_assertion
+        self.equivalent_object_property = equivalent_object_property
+        self.inverse_object_properties = inverse_object_properties
+        self.object_property_characteristic = object_property_characteristic
+        self.sub_object_property = sub_object_property
+        self.object_property_range = object_property_range
+        self.object_property_domain = object_property_domain
         self.data_graph_iri = data_graph_iri
         self.ontology_graph_iri = ontology_graph_iri
-        self.result_graph_iri = result_graph_iri
+        self.output_graph_iri = output_graph_iri
         self.reasoner = reasoner
         self.max_ram_percentage = max_ram_percentage
         self.temp = f"reason_{uuid4().hex}"
@@ -260,14 +274,12 @@ def reason(self, graphs: dict) -> None:
             f"--exclude-external-entities "
             f"reduce --reasoner {self.reasoner} "
             f'unmerge --input "{data_location}" '
-            f'annotate --ontology-iri "{self.result_graph_iri}" '
+            f'annotate --ontology-iri "{self.output_graph_iri}" '
             f"--remove-annotations "
             f'--language-annotation rdfs:label "Eccenca Reasoning Result {utctime}" en '
             f"--language-annotation rdfs:comment "
             f'"Reasoning result set of <{self.data_graph_iri}> and '
             f'<{self.ontology_graph_iri}>" en '
-            f"--language-annotation prov:wasGeneratedBy "
-            f'"cmem-plugin-reason ({self.reasoner})" en '
             f'--link-annotation prov:wasDerivedFrom "{self.data_graph_iri}" '
             f"--link-annotation prov:wasDerivedFrom "
             f'"{self.ontology_graph_iri}" '
@@ -290,5 +302,6 @@ def execute(self, inputs: tuple, context: ExecutionContext) -> None:  # noqa: AR
         create_xml_catalog_file(self.temp, graphs)
         self.reason(graphs)
         setup_cmempy_user_access(context.user)
-        send_result(self.result_graph_iri, Path(self.temp) / "result.ttl")
-        remove_temp(self, ["catalog-v001.xml", "result.ttl", *graphs.values()])
+        send_result(self.output_graph_iri, Path(self.temp) / "result.ttl")
+        post_provenance(self, context)
+        remove_temp(self)

cmem_plugin_reason/plugin_validate.py

@@ -32,16 +32,21 @@
     ROBOT,
     create_xml_catalog_file,
     get_graphs_tree,
+    post_provenance,
     remove_temp,
     send_result,
 )
 
 
 @Plugin(
-    label="Validate ontology consistency",
-    description="",
-    documentation="""""",
-    icon=Icon(package=__package__, file_name="obofoundry.png"),
+    label="Validate",
+    description="Validates the consistency of an OWL ontology.",
+    documentation="""A task validating the consistency of an OWL ontology and generating an
+    explanation if inconsistencies are found. The explanation can be written to the project as a
+    Markdown file and/or to a specified graph. The Markdown string is also provided as an output
+    entity using the path "text". The following reasoners are supported: ELK, Expression
+    Materializing Reasoner, HermiT, JFact, Structural Reasoner and Whelk.""",
+    icon=Icon(package=__package__, file_name="validate.png"),
     parameters=[
         REASONER_PARAMETER,
         ONTOLOGY_GRAPH_IRI_PARAMETER,
@@ -50,8 +55,7 @@
             param_type=BoolParameterType(),
             name="write_md",
             label="Write Markdown explanation file",
-            description="Write Markdown file with explanation to project. ⚠️ Existing files will "
-            "be overwritten.",
+            description="Write Markdown file with explanation to project.",
             default_value=False,
         ),
         PluginParameter(
@@ -73,7 +77,7 @@
             name="md_filename",
             label="Output filename",
             description="The filename of the Markdown file with the explanation of "
-            "inconsistencies.",
+            "inconsistencies.⚠️ Existing files will be overwritten.",
         ),
         PluginParameter(
             param_type=BoolParameterType(),
@@ -151,8 +155,6 @@ def validate(self, graphs: dict) -> None:
                 f'--language-annotation rdfs:label "Ontology Validation Result {utctime}" en '
                 f"--language-annotation rdfs:comment "
                 f'"Ontology validation of <{self.ontology_graph_iri}>" en '
-                f"--language-annotation prov:wasGeneratedBy "
-                f'"cmem-plugin-validate ({self.reasoner})" en '
                 f'--link-annotation prov:wasDerivedFrom "{self.ontology_graph_iri}" '
                 f'--typed-annotation dc:created "{utctime}" xsd:dateTime '
                 f'--output "{self.temp}/output.ttl"'
@@ -182,26 +184,21 @@ def execute(self, inputs: tuple, context: ExecutionContext) -> Entities | None:
         self.get_graphs(graphs, context)
         create_xml_catalog_file(self.temp, graphs)
         self.validate(graphs)
-        files = ["catalog-v001.xml", self.md_filename, *graphs.values()]
-        if self.produce_graph:
-            files.append("output.ttl")
-
-        text = (Path(self.temp) / self.md_filename).read_text()
-        if text == "No explanations found.":
-            remove_temp(self, files)
-            return None
 
         if self.produce_graph:
             setup_cmempy_user_access(context.user)
             send_result(self.output_graph_iri, Path(self.temp) / "output.ttl")
+            setup_cmempy_user_access(context.user)
+            post_provenance(self, context)
 
         if self.write_md:
             setup_cmempy_user_access(context.user)
             self.make_resource(context)
+        text = (Path(self.temp) / self.md_filename).read_text()
 
-        remove_temp(self, files)
+        remove_temp(self)
 
-        if self.stop_at_inconsistencies:
+        if self.stop_at_inconsistencies and text != "No explanations found.":
             raise RuntimeError("Inconsistencies found in Ontology.")
 
         entities = [

cmem_plugin_reason/utils.py

@@ -1,16 +1,21 @@
 """Common constants and functions"""
 
+import json
 import re
 import unicodedata
 from collections import OrderedDict
 from pathlib import Path
+from secrets import token_hex
+from shutil import rmtree
 from xml.etree.ElementTree import Element, SubElement, tostring
 
 from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, post_streamed
+from cmem.cmempy.dp.proxy.sparql import post as post_select
+from cmem.cmempy.dp.proxy.update import post as post_update
 from cmem_plugin_base.dataintegration.description import PluginParameter
 from cmem_plugin_base.dataintegration.parameter.choice import ChoiceParameterType
 from cmem_plugin_base.dataintegration.parameter.graph import GraphParameterType
-from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
+from cmem_plugin_base.dataintegration.plugins import ExecutionContext, WorkflowPlugin
 from cmem_plugin_base.dataintegration.types import IntParameterType
 from defusedxml import minidom
 
@@ -108,14 +113,75 @@ def send_result(iri: str, filepath: Path) -> None:
     )
 
 
-def remove_temp(plugin: WorkflowPlugin, files: list) -> None:
-    """Remove temproray files"""
-    for file in files:
-        try:
-            (Path(plugin.temp) / file).unlink()
-        except (OSError, FileNotFoundError) as err:
-            plugin.log.warning(f"Cannot remove file {file} ({err})")
+def remove_temp(plugin: WorkflowPlugin) -> None:
+    """Remove temporary files"""
     try:
-        Path(plugin.temp).rmdir()
+        rmtree(plugin.temp)
     except (OSError, FileNotFoundError) as err:
         plugin.log.warning(f"Cannot remove directory {plugin.temp} ({err})")
+
+
+def post_provenance(plugin: WorkflowPlugin, context: ExecutionContext) -> None:
+    """Insert provenance"""
+    plugin_iri = (
+        f"http://dataintegration.eccenca.com/{context.task.project_id()}/{context.task.task_id()}"
+    )
+    project_graph = f"http://di.eccenca.com/project/{context.task.project_id()}"
+
+    type_query = f"""
+        SELECT ?type ?label {{
+            GRAPH <{project_graph}> {{
+                <{plugin_iri}> a ?type .
+                <{plugin_iri}> <http://www.w3.org/2000/01/rdf-schema#label> ?label .
+                FILTER(STRSTARTS(STR(?type), "https://vocab.eccenca.com/di/functions/"))
+            }}
+        }}
+    """
+
+    result = json.loads(post_select(query=type_query))
+
+    try:
+        plugin_type = result["results"]["bindings"][0]["type"]["value"]
+    except IndexError:
+        plugin.log.warning("Could not add provenance data to output graph.")
+        return
+    plugin_label = result["results"]["bindings"][0]["label"]["value"]
+
+    param_split = (
+        plugin_type.replace(
+            "https://vocab.eccenca.com/di/functions/Plugin_",
+            "https://vocab.eccenca.com/di/functions/param_",
+        )
+        + "_"
+    )
+
+    parameter_query = f"""
+        SELECT ?parameter {{
+            GRAPH <{project_graph}> {{
+                <{plugin_iri}> ?parameter ?o .
+                FILTER(STRSTARTS(STR(?parameter), "https://vocab.eccenca.com/di/functions/param_"))
+            }}
+        }}
+    """
+
+    new_plugin_iri = f'{"_".join(plugin_iri.split("_")[:-1])}_{token_hex(8)}'
+    result = json.loads(post_select(query=parameter_query))
+    param_sparql = ""
+    for binding in result["results"]["bindings"]:
+        param_iri = binding["parameter"]["value"]
+        param_val = plugin.__dict__[binding["parameter"]["value"].split(param_split)[1]]
+        param_sparql += f'\n<{new_plugin_iri}> <{param_iri}> "{param_val}" .'
+
+    insert_query = f"""
+        INSERT DATA {{
+            GRAPH <{plugin.output_graph_iri}> {{
+                <{plugin.output_graph_iri}> <http://www.w3.org/ns/prov#wasGeneratedBy>
+                    <{new_plugin_iri}> .
+                <{new_plugin_iri}> a <{plugin_type}>, <https://vocab.eccenca.com/di/CustomTask> .
+                <{new_plugin_iri}> <http://www.w3.org/2000/01/rdf-schema#label> "{plugin_label}" .
+                {param_sparql}
+            }}
+        }}
+    """
+
+    post_update(query=insert_query)