From 582cf792c59b788b35b602c2e77ca18cbd326019 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Tue, 1 Oct 2024 14:41:09 -0400 Subject: [PATCH] Demo: started adding Resiliency, fixed some Spec issues --- demo/scenarios/1_requirements.ipynb | 22 ++++++++----------- demo/scenarios/properties/accuracy.py | 6 ++--- demo/scenarios/properties/interoperability.py | 1 - demo/scenarios/properties/monitorability.py | 10 ++++----- demo/scenarios/properties/resilience.py | 17 ++++++++++++++ 5 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 demo/scenarios/properties/resilience.py diff --git a/demo/scenarios/1_requirements.ipynb b/demo/scenarios/1_requirements.ipynb index 09156e76..b44026fc 100644 --- a/demo/scenarios/1_requirements.ipynb +++ b/demo/scenarios/1_requirements.ipynb @@ -211,7 +211,7 @@ " SystemDescriptor,\n", " GoalDescriptor,\n", " ProblemType,\n", - " RiskDescriptor\n", + " RiskDescriptor,\n", ")\n", "\n", "card = NegotiationCard(\n", @@ -542,6 +542,7 @@ "from values.multiple_ranksums import MultipleRanksums\n", "from properties.monitorability import Monitorability\n", "from properties.interoperability import Interoperability\n", + "from properties.resilience import Resilience\n", "from properties.accuracy import Accuracy\n", "from values.string import String\n", "\n", @@ -582,7 +583,9 @@ " ): {\n", " \"predicting cpu\": LocalProcessCPUUtilization.value().max_utilization_less_than(\n", " 30.0\n", - " )\n", + " ),\n", + " # Performance: Inference time on Operational Platform\n", + " \"predicting cpu time\": Real.less_than(2.0),\n", " },\n", " Interpretability(\"Important to understand what the model is doing\"): {\n", " \"image attributions\": Image.ignore(\"Inspect the image.\")\n", @@ -605,20 +608,13 @@ " \"Model - Input Validation Error\"\n", " )\n", " },\n", - " # Monitorability: Detect OOD inputs\n", - " Monitorability(\"Monitor inputs for OOD data\"): {\n", - " \"detect ood inputs\": String.contains(\"Model - Input OOD Error\")\n", - " },\n", - " # Monitorability: Detect output shifts\n", - " Monitorability(\"Monitor outputs for unexpected shifts\"): {\n", + " # Monitorability: Detect OOD inputs and unexpected shifts\n", + " Monitorability(\"Monitor inputs for OOD data and unexpected shifts\"): {\n", + " \"detect ood inputs\": String.contains(\"Model - Input OOD Error\"),\n", " \"monitor output confidence shift\": String.contains(\n", " \"Model - Output Confidence Error\"\n", - " )\n", + " ),\n", " },\n", - " # Performance: Inference time on Operational Platform\n", - " PredictingComputeCost(\n", - " \"End to end latency for inference on operational platform\"\n", - " ): {\"predicting cpu time\": Real.less_than(2.0)},\n", " }\n", ")\n", "spec.save(parents=True, force=True)" diff --git a/demo/scenarios/properties/accuracy.py b/demo/scenarios/properties/accuracy.py index 6d73b419..7e3651b2 100644 --- a/demo/scenarios/properties/accuracy.py +++ b/demo/scenarios/properties/accuracy.py @@ -11,8 +11,8 @@ def __init__(self, rationale: str): super().__init__( instance=self, description=""" - The Accuracy property assesses the total predictive performance requirements of a - model and system. These requirements are typically given as a single real valued number. - """, + The Accuracy property assesses the total predictive performance requirements of a + model and system. These requirements are typically given as a single real valued number. + """, rationale=rationale, ) diff --git a/demo/scenarios/properties/interoperability.py b/demo/scenarios/properties/interoperability.py index b25ba38e..7ef2ea77 100644 --- a/demo/scenarios/properties/interoperability.py +++ b/demo/scenarios/properties/interoperability.py @@ -14,7 +14,6 @@ def __init__(self, rationale: str): The Interoperability property evaluates the requirements on the interfaces to a model and system. These requirements may be expressed in a variety of ways, including requriements on line formats in log files, timestamps, and log content. - """, rationale=rationale, ) diff --git a/demo/scenarios/properties/monitorability.py b/demo/scenarios/properties/monitorability.py index ecbe9876..95b79096 100644 --- a/demo/scenarios/properties/monitorability.py +++ b/demo/scenarios/properties/monitorability.py @@ -3,17 +3,17 @@ class Monitorability(Property): """ - The Interoperability property reflects contract requirements on model interfaces. + The Monitorability property reflects monitoring requirements. """ def __init__(self, rationale: str): - """Initialize a Interoperability instance.""" + """Initialize a Monitorability instance.""" super().__init__( instance=self, description=""" - The Monitorability property assesses the monitoring requirements of a - model and system. These requirements may be expressed in a variety of ways, - including requriements on line formats in log files, timestamps, and log content. + The Monitorability property assesses the monitoring requirements of a + model and system. These requirements may be expressed in a variety of ways, + including requriements on line formats in log files, timestamps, and log content. """, rationale=rationale, ) diff --git a/demo/scenarios/properties/resilience.py b/demo/scenarios/properties/resilience.py new file mode 100644 index 00000000..8fee7814 --- /dev/null +++ b/demo/scenarios/properties/resilience.py @@ -0,0 +1,17 @@ +from mlte.property.base import Property + + +class Resilience(Property): + """ + The Resilience property reflects ... + """ + + def __init__(self, rationale: str): + """Initialize a Resilience instance.""" + super().__init__( + instance=self, + description=""" + The Resilience property assesses ... + """, + rationale=rationale, + )