Backend, property > quality attribute category

mlte-team · Dec 10, 2024 · 680213c · 680213c
1 parent 4064f72
commit 680213c
Show file tree

Hide file tree

Showing 67 changed files with 635 additions and 541 deletions.
diff --git a/demo/scenarios/1_requirements.ipynb b/demo/scenarios/1_requirements.ipynb
@@ -69,9 +69,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mRunning cells with 'mlte-python-3NUZF8_b-py3.9 (Python 3.9.20)' requires the ipykernel package.\n",
+      "\u001b[1;31mRun the following command to install 'ipykernel' into the Python environment. \n",
+      "\u001b[1;31mCommand: '/Users/aderr/Library/Caches/pypoetry/virtualenvs/mlte-python-3NUZF8_b-py3.9/bin/python -m pip install ipykernel -U --force-reinstall'"
+     ]
+    }
+   ],
    "source": [
     "import os\n",
     "from mlte.session import set_context, set_store\n",
@@ -96,7 +107,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -208,7 +219,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -496,7 +507,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -527,24 +538,37 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "In MLTE, we define requirements by constructing a specification (`Spec`). For each property, we define the validations to perform as well. Note that several new `Value` types (`MultipleAccuracy`, `RankSums`, `MultipleRanksums`) had to be created to define the validation methods that will validate each Condition."
+    "In MLTE, we define requirements by constructing a specification (`Spec`). For each qa category, we define the validations to perform as well. Note that several new `Value` types (`MultipleAccuracy`, `RankSums`, `MultipleRanksums`) had to be created to define the validation methods that will validate each Condition."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'numpy'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 19\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdemo\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mscenarios\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mvalues\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmultiple_accuracy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MultipleAccuracy\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdemo\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mscenarios\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mvalues\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mranksums\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RankSums\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdemo\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mscenarios\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mvalues\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmultiple_ranksums\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MultipleRanksums\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdemo\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mscenarios\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mproperties\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmonitorability\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Monitorability\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdemo\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mscenarios\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mproperties\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01minteroperability\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Interoperability\n",
+      "File \u001b[0;32m~/repos/mlte/mlte-514/demo/scenarios/values/multiple_ranksums.py:8\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m__future__\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m annotations\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Any, Dict\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmlte\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mevidence\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetadata\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m EvidenceMetadata\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmlte\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mspec\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcondition\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Condition\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'numpy'"
+     ]
+    }
+   ],
    "source": [
     "from mlte.spec.spec import Spec\n",
     "\n",
     "# The Properties we want to validate, associated with our scenarios.\n",
-    "from mlte.property.costs.storage_cost import StorageCost\n",
-    "from mlte.property.fairness.fairness import Fairness\n",
-    "from mlte.property.robustness.robustness import Robustness\n",
-    "from mlte.property.interpretability.interpretability import Interpretability\n",
-    "from mlte.property.costs.predicting_memory_cost import PredictingMemoryCost\n",
-    "from mlte.property.costs.predicting_compute_cost import PredictingComputeCost\n",
+    "from mlte.qa_category.costs.storage_cost import StorageCost\n",
+    "from mlte.qa_category.fairness.fairness import Fairness\n",
+    "from mlte.qa_category.robustness.robustness import Robustness\n",
+    "from mlte.qa_category.interpretability.interpretability import Interpretability\n",
+    "from mlte.qa_category.costs.predicting_memory_cost import PredictingMemoryCost\n",
+    "from mlte.qa_category.costs.predicting_compute_cost import PredictingComputeCost\n",
     "\n",
     "# The Value types we will use to validate each condition.\n",
     "from mlte.measurement.storage import LocalObjectSize\n",
@@ -562,9 +586,9 @@
     "from demo.scenarios.values.string import String\n",
     "\n",
     "\n",
-    "# The full spec. Note that the Robustness Property contains conditions for both Robustness scenarios.\n",
+    "# The full spec. Note that the Robustness QualityAttributCategory  contains conditions for both Robustness scenarios.\n",
     "spec = Spec(\n",
-    "    properties={\n",
+    "    qa_categories={\n",
     "        Fairness(\n",
     "            \"Important check if model performs well accross different populations\"\n",
     "        ): {\n",
@@ -637,11 +661,18 @@
     ")\n",
     "spec.save(parents=True, force=True)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "mlte-python-3NUZF8_b-py3.9",
    "language": "python",
    "name": "python3"
   },
@@ -655,12 +686,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "82adda432962015d5f71beb9387a99f24d390514e497c776c87ff3434daf7312"
-   }
+   "version": "3.9.20"
   }
  },
  "nbformat": 4,

diff --git a/demo/scenarios/2_evidence.ipynb b/demo/scenarios/2_evidence.ipynb
@@ -7,7 +7,7 @@
    "source": [
     "## 2. Collect Evidence\n",
     "\n",
-    "In the second phase of SDMT, we collect _evidence_ to attest to the fact that the model realized the properties specified in the previous phase.\n",
+    "In the second phase of SDMT, we collect _evidence_ to attest to the fact that the model realized the QA categories specified in the previous phase.\n",
     "\n",
     "We define and instantiate `Measurement`s to generate this evidence. Each individual piece of evidence is a `Value`. Once `Value`s are produced, we can persist them to an _artifact store_ to maintain our evidence across sessions. "
    ]

diff --git a/demo/scenarios/properties/accuracy.py b/demo/scenarios/properties/accuracy.py
@@ -1,17 +1,17 @@
-from mlte.property.base import Property
+from mlte.qa_category.base import QACategory
 
 
-class Accuracy(Property):
+class Accuracy(QACategory):
     """
-    The Accuracy property reflects the overall functional accuracy of a model.
+    The Accuracy qa category reflects the overall functional accuracy of a model.
     """
 
     def __init__(self, rationale: str):
         """Initialize a Accuracy instance."""
         super().__init__(
             instance=self,
             description="""
-                The Accuracy property assesses the total predictive performance requirements of a
+                The Accuracy qa category assesses the total predictive performance requirements of a
                 model and system. These requirements are typically given as a single real valued number. 
                 """,
             rationale=rationale,

diff --git a/demo/scenarios/properties/interoperability.py b/demo/scenarios/properties/interoperability.py
@@ -1,18 +1,18 @@
-from mlte.property.base import Property
+from mlte.qa_category.base import QACategory
 
 
-class Interoperability(Property):
+class Interoperability(QACategory):
     """
-    The Interoperability property reflects contract requirements on model interfaces.
+    The Interoperability qa category reflects contract requirements on model interfaces.
     """
 
     def __init__(self, rationale: str):
         """Initialize a Interoperability instance."""
         super().__init__(
             instance=self,
             description="""
-                The Interoperability property evaluates the requirements on the interfaces to a model and system. 
-                These requirements may be expressed in a variety of ways,
+                The Interoperability qa category evaluates the requirements on the 
+                interfaces to a model and system. These requirements may be expressed in a variety of ways,
                 including requriements on line formats in log files, timestamps, and log content. 
                 """,
             rationale=rationale,

diff --git a/demo/scenarios/properties/monitorability.py b/demo/scenarios/properties/monitorability.py
@@ -1,17 +1,17 @@
-from mlte.property.base import Property
+from mlte.qa_category.base import QACategory
 
 
-class Monitorability(Property):
+class Monitorability(QACategory):
     """
-    The Monitorability property reflects  monitoring requirements.
+    The Monitorability qa category reflects  monitoring requirements.
     """
 
     def __init__(self, rationale: str):
         """Initialize a Monitorability instance."""
         super().__init__(
             instance=self,
             description="""
-                The Monitorability property assesses the monitoring requirements of a
+                The Monitorability qa category assesses the monitoring requirements of a
                 model and system. These requirements may be expressed in a variety of ways,
                 including requriements on line formats in log files, timestamps, and log content. 
                 """,

diff --git a/demo/scenarios/properties/resilience.py b/demo/scenarios/properties/resilience.py
@@ -1,19 +1,19 @@
-from mlte.property.base import Property
+from mlte.qa_category.base import QACategory
 
 
-class Resilience(Property):
+class Resilience(QACategory):
     """
-    The Resilience property reflects ...
+    The Resilience qa category reflects ...
     """
 
     def __init__(self, rationale: str):
         """Initialize a Resilience instance."""
         super().__init__(
             instance=self,
             description="""
-                The Resilience property assesses if a component is able to continue to carry out its mission in 
-                the face of adversity (i.e., if it provides required capabilities despite excessive stresses 
-                that can cause disruptions).
+                The Resilience qa category assesses if a component is able to continue 
+                to carry out its mission in the face of adversity (i.e., if it provides required
+                capabilities despite excessive stresses that can cause disruptions).
                 """,
             rationale=rationale,
         )
diff --git a/demo/simple/evidence.ipynb b/demo/simple/evidence.ipynb
@@ -7,7 +7,7 @@
    "source": [
     "## 2. Collect Evidence\n",
     "\n",
-    "In the second phase of SDMT, we collect _evidence_ to attest to the fact that the model realized the properties specified in the previous phase.\n",
+    "In the second phase of SDMT, we collect _evidence_ to attest to the fact that the model realized the QA categories specified in the previous phase.\n",
     "\n",
     "We define and instantiate `Measurement`s to generate this evidence. Each individual piece of evidence is a `Value`. Once `Value`s are produced, we can persist them to an _artifact store_ to maintain our evidence across sessions. "
    ]

diff --git a/demo/simple/requirements.ipynb b/demo/simple/requirements.ipynb
@@ -45,21 +45,21 @@
    "source": [
     "#### Build a `Specification`\n",
     "\n",
-    "In MLTE, we define requirements by constructing a specification (`Spec`). For each property, we define the validations to perform as well."
+    "In MLTE, we define requirements by constructing a specification (`Spec`). For each qa category, we define the validations to perform as well."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "from mlte.spec.spec import Spec\n",
     "\n",
-    "from mlte.property.costs.storage_cost import StorageCost\n",
-    "from mlte.property.costs.training_memory_cost import TrainingMemoryCost\n",
-    "from mlte.property.costs.training_compute_cost import TrainingComputeCost\n",
-    "from mlte.property.functionality.task_efficacy import TaskEfficacy\n",
+    "from mlte.qa_category.costs.storage_cost import StorageCost\n",
+    "from mlte.qa_category.costs.training_memory_cost import TrainingMemoryCost\n",
+    "from mlte.qa_category.costs.training_compute_cost import TrainingComputeCost\n",
+    "from mlte.qa_category.functionality.task_efficacy import TaskEfficacy\n",
     "\n",
     "\n",
     "from mlte.measurement.storage import LocalObjectSize\n",
@@ -70,7 +70,7 @@
     "from mlte.value.types.image import Image\n",
     "\n",
     "spec = Spec(\n",
-    "    properties={\n",
+    "    qa_categories={\n",
     "        TaskEfficacy(\n",
     "            \"Important to understand if the model is useful for this case\"\n",
     "        ): {\n",

diff --git a/docs/docs/using_mlte.md b/docs/docs/using_mlte.md
@@ -38,8 +38,8 @@ A `Specification` (or `Spec`) represents the requirements the model must meet in
 
 ```python
 from mlte.spec.spec import Spec
-from mlte.property.costs.storage_cost import StorageCost
-from mlte.property.fairness.fairness import Fairness
+from mlte.qa_category.costs.storage_cost import StorageCost
+from mlte.qa_category.fairness.fairness import Fairness
 
 from mlte.measurement.storage import LocalObjectSize
 from demo.scenarios.values.multiple_accuracy import MultipleAccuracy
@@ -198,10 +198,10 @@ A `Spec` represents the requirements and corresponding thresholds (or validators
 from mlte.spec.spec import Spec
 
 # The Properties we want to validate, associated with our scenarios.
-from mlte.property.costs.storage_cost import StorageCost
-from mlte.property.fairness.fairness import Fairness
-from mlte.property.robustness.robustness import Robustness
-from mlte.property.costs.predicting_memory_cost import PredictingMemoryCost
+from mlte.qa_category.costs.storage_cost import StorageCost
+from mlte.qa_category.fairness.fairness import Fairness
+from mlte.qa_category.robustness.robustness import Robustness
+from mlte.qa_category.costs.predicting_memory_cost import PredictingMemoryCost
 
 # The Value types we will use to validate each condition.
 from mlte.measurement.storage import LocalObjectSize

diff --git a/mlte/catalog/model.py b/mlte/catalog/model.py
@@ -53,11 +53,11 @@ class CatalogEntry(Filtrable):
     tags: List[str] = []
     """Tags for the problem for the entry."""
 
-    property_category: Optional[str] = None
-    """The property category or QA for the entry."""
+    qa_category: Optional[str] = None
+    """The qa category for the entry."""
 
-    property: Optional[str] = None
-    """The property for the entry."""
+    quality_attribute: Optional[str] = None
+    """The quality attribute for the entry."""
 
     code_type: CatalogEntryType
     """The code type the entry."""

diff --git a/mlte/frontend/nuxt-app/assets/schema/artifact/spec/v0.0.1/schema.json b/mlte/frontend/nuxt-app/assets/schema/artifact/spec/v0.0.1/schema.json
@@ -1,7 +1,7 @@
 {
   "$defs": {
     "ConditionModel": {
-      "description": "A description of a condition for a property.",
+      "description": "A description of a condition for a qa category.",
       "properties": {
         "name": {
           "title": "Name",
@@ -30,8 +30,8 @@
       "title": "ConditionModel",
       "type": "object"
     },
-    "PropertyModel": {
-      "description": "A description of a property.",
+    "QACategoryModel": {
+      "description": "A description of a qa category.",
       "properties": {
         "name": {
           "title": "Name",
@@ -78,7 +78,7 @@
         "name",
         "module"
       ],
-      "title": "PropertyModel",
+      "title": "QACategoryModel",
       "type": "object"
     }
   },
@@ -96,7 +96,7 @@
     "properties": {
       "type": "array",
       "items": {
-        "$ref": "#/$defs/PropertyModel"
+        "$ref": "#/$defs/QACategoryModel"
       },
       "default": [],
       "title": "Properties"