Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin' into feature/catalog
Browse files Browse the repository at this point in the history
  • Loading branch information
@sei-aderr
sei-aderr committed Sep 20, 2024
2 parents 25d625d + db66701 commit 55a61be
Show file tree
Hide file tree
Showing 29 changed files with 1,938 additions and 1,645 deletions.
47 changes: 20 additions & 27 deletions demo/scenarios/0_requirements.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,9 @@
" FieldDescriptor,\n",
" LabelDescriptor,\n",
" ModelDescriptor,\n",
" ModelDevelopmentDescriptor,\n",
" ModelResourcesDescriptor,\n",
" ModelProductionDescriptor,\n",
" ModelInterfaceDescriptor,\n",
" ModelIODescriptor,\n",
" QASDescriptor,\n",
")\n",
"from mlte.negotiation.model import (\n",
" SystemDescriptor,\n",
" GoalDescriptor,\n",
" ProblemType,\n",
Expand Down Expand Up @@ -154,28 +149,26 @@
" )\n",
" ],\n",
" model=ModelDescriptor(\n",
" development=ModelDevelopmentDescriptor(\n",
" resources=ModelResourcesDescriptor(\n",
" cpu=\"1\", gpu=\"0\", memory=\"6MiB\", storage=\"2KiB\"\n",
" )\n",
" development_compute_resources=ModelResourcesDescriptor(\n",
" cpu=\"1\", gpu=\"0\", memory=\"6MiB\", storage=\"2KiB\"\n",
" ),\n",
" production=ModelProductionDescriptor(\n",
" deployment_platform=\"local server\",\n",
" capability_deployment_mechanism=\"API\",\n",
" interface=ModelInterfaceDescriptor(\n",
" input=ModelIODescriptor(\n",
" name=\"i1\", description=\"description\", type=\"Vector[150]\"\n",
" ),\n",
" output=ModelIODescriptor(\n",
" name=\"o1\", description=\"description\", type=\"Vector[3]\"\n",
" ),\n",
" ),\n",
" resources=ModelResourcesDescriptor(\n",
" cpu=\"1\",\n",
" gpu=\"0\",\n",
" memory=\"6MiB\",\n",
" storage=\"2KiB\",\n",
" ),\n",
" deployment_platform=\"local server\",\n",
" capability_deployment_mechanism=\"API\",\n",
" input_specification=[\n",
" ModelIODescriptor(\n",
" name=\"i1\", description=\"description\", type=\"Vector[150]\"\n",
" )\n",
" ],\n",
" output_specification=[\n",
" ModelIODescriptor(\n",
" name=\"o1\", description=\"description\", type=\"Vector[3]\"\n",
" )\n",
" ],\n",
" production_compute_resources=ModelResourcesDescriptor(\n",
" cpu=\"1\",\n",
" gpu=\"0\",\n",
" memory=\"6MiB\",\n",
" storage=\"2KiB\",\n",
" ),\n",
" ),\n",
" qas=[\n",
Expand Down Expand Up @@ -266,7 +259,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down
88 changes: 5 additions & 83 deletions demo/scenarios/2_report.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,96 +124,15 @@
"metadata": {},
"outputs": [],
"source": [
"from mlte.model.shared import (\n",
" ProblemType,\n",
" GoalDescriptor,\n",
" MetricDescriptor,\n",
" ModelProductionDescriptor,\n",
" ModelInterfaceDescriptor,\n",
" ModelIODescriptor,\n",
" ModelResourcesDescriptor,\n",
" RiskDescriptor,\n",
" DataDescriptor,\n",
" DataClassification,\n",
" FieldDescriptor,\n",
" LabelDescriptor,\n",
")\n",
"from mlte.report.artifact import (\n",
" Report,\n",
" SummaryDescriptor,\n",
" PerformanceDesciptor,\n",
" IntendedUseDescriptor,\n",
" CommentDescriptor,\n",
" QuantitiveAnalysisDescriptor,\n",
")\n",
"from mlte.negotiation.artifact import NegotiationCard\n",
"\n",
"report = Report(\n",
" summary=SummaryDescriptor(\n",
" problem_type=ProblemType.CLASSIFICATION, task=\"Flower classification\"\n",
" ),\n",
" performance=PerformanceDesciptor(\n",
" goals=[\n",
" GoalDescriptor(\n",
" description=\"The model should perform well.\",\n",
" metrics=[\n",
" MetricDescriptor(\n",
" description=\"accuracy\",\n",
" baseline=\"Better than random chance.\",\n",
" )\n",
" ],\n",
" )\n",
" ],\n",
" validated_spec_id=validated_spec.identifier,\n",
" ),\n",
" intended_use=IntendedUseDescriptor(\n",
" usage_context=\"A handheld flower identification device.\",\n",
" production_requirements=ModelProductionDescriptor(\n",
" deployment_platform=\"local server\",\n",
" capability_deployment_mechanism=\"API\",\n",
" interface=ModelInterfaceDescriptor(\n",
" input=ModelIODescriptor(\n",
" name=\"i1\", description=\"description\", type=\"Vector[150]\"\n",
" ),\n",
" output=ModelIODescriptor(\n",
" name=\"o1\", description=\"description\", type=\"Vector[3]\"\n",
" ),\n",
" ),\n",
" resources=ModelResourcesDescriptor(\n",
" cpu=\"1\", gpu=\"0\", memory=\"6MiB\", storage=\"2KiB\"\n",
" ),\n",
" ),\n",
" ),\n",
" risks=RiskDescriptor(\n",
" fp=\"The wrong type of flower is identified.\",\n",
" fn=\"The flower is not identified.\",\n",
" other=\"N/A\",\n",
" ),\n",
" data=[\n",
" DataDescriptor(\n",
" description=\"Flower dataset.\",\n",
" classification=DataClassification.UNCLASSIFIED,\n",
" access=\"None\",\n",
" labeling_method=\"by hand\",\n",
" fields=[\n",
" FieldDescriptor(\n",
" name=\"Sepal length\",\n",
" description=\"The length of the sepal.\",\n",
" type=\"float\",\n",
" expected_values=\"N/A\",\n",
" missing_values=\"N/A\",\n",
" special_values=\"N/A\",\n",
" )\n",
" ],\n",
" labels=[\n",
" LabelDescriptor(description=\"Dahlia\", percentage=30.0),\n",
" LabelDescriptor(description=\"Sunflower\", percentage=30.0),\n",
" LabelDescriptor(description=\"Azalea\", percentage=40.0),\n",
" ],\n",
" policies=\"N/A\",\n",
" rights=\"N/A\",\n",
" source=\"https://www.robots.ox.ac.uk/~vgg/data/flowers/102/\",\n",
" )\n",
" ],\n",
" validated_spec_id=validated_spec.identifier,\n",
" comments=[\n",
" CommentDescriptor(\n",
" content=\"This model should not be used for nefarious purposes.\"\n",
Expand All @@ -224,6 +143,9 @@
" ),\n",
")\n",
"\n",
"negotiation_card = NegotiationCard.load()\n",
"report = report.populate_from(negotiation_card)\n",
"\n",
"report.save(force=True, parents=True)"
]
}
Expand Down
45 changes: 19 additions & 26 deletions demo/simple/negotiation.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,8 @@
" FieldDescriptor,\n",
" LabelDescriptor,\n",
" ModelDescriptor,\n",
" ModelDevelopmentDescriptor,\n",
" ModelResourcesDescriptor,\n",
" ModelProductionDescriptor,\n",
" ModelInterfaceDescriptor,\n",
" ModelIODescriptor,\n",
")\n",
"from mlte.negotiation.model import (\n",
" SystemDescriptor,\n",
" GoalDescriptor,\n",
" ProblemType,\n",
Expand Down Expand Up @@ -129,28 +124,26 @@
" )\n",
" ],\n",
" model=ModelDescriptor(\n",
" development=ModelDevelopmentDescriptor(\n",
" resources=ModelResourcesDescriptor(\n",
" cpu=\"1\", gpu=\"0\", memory=\"6MiB\", storage=\"2KiB\"\n",
" )\n",
" development_compute_resources=ModelResourcesDescriptor(\n",
" cpu=\"1\", gpu=\"0\", memory=\"6MiB\", storage=\"2KiB\"\n",
" ),\n",
" production=ModelProductionDescriptor(\n",
" deployment_platform=\"local server\",\n",
" capability_deployment_mechanism=\"API\",\n",
" interface=ModelInterfaceDescriptor(\n",
" input=ModelIODescriptor(\n",
" name=\"i1\", description=\"description\", type=\"Vector[150]\"\n",
" ),\n",
" output=ModelIODescriptor(\n",
" name=\"o1\", description=\"description\", type=\"Vector[3]\"\n",
" ),\n",
" ),\n",
" resources=ModelResourcesDescriptor(\n",
" cpu=\"1\",\n",
" gpu=\"0\",\n",
" memory=\"6MiB\",\n",
" storage=\"2KiB\",\n",
" ),\n",
" deployment_platform=\"local server\",\n",
" capability_deployment_mechanism=\"API\",\n",
" input_specification=[\n",
" ModelIODescriptor(\n",
" name=\"i1\", description=\"description\", type=\"Vector[150]\"\n",
" )\n",
" ],\n",
" output_specification=[\n",
" ModelIODescriptor(\n",
" name=\"o1\", description=\"description\", type=\"Vector[3]\"\n",
" )\n",
" ],\n",
" production_compute_resources=ModelResourcesDescriptor(\n",
" cpu=\"1\",\n",
" gpu=\"0\",\n",
" memory=\"6MiB\",\n",
" storage=\"2KiB\",\n",
" ),\n",
" ),\n",
")\n",
Expand Down
98 changes: 10 additions & 88 deletions demo/simple/report.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -44,7 +44,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -74,7 +74,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -92,7 +92,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -115,100 +115,19 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from mlte.model.shared import (\n",
" ProblemType,\n",
" GoalDescriptor,\n",
" MetricDescriptor,\n",
" ModelProductionDescriptor,\n",
" ModelInterfaceDescriptor,\n",
" ModelIODescriptor,\n",
" ModelResourcesDescriptor,\n",
" RiskDescriptor,\n",
" DataDescriptor,\n",
" DataClassification,\n",
" FieldDescriptor,\n",
" LabelDescriptor,\n",
")\n",
"from mlte.report.artifact import (\n",
" Report,\n",
" SummaryDescriptor,\n",
" PerformanceDesciptor,\n",
" IntendedUseDescriptor,\n",
" CommentDescriptor,\n",
" QuantitiveAnalysisDescriptor,\n",
")\n",
"from mlte.negotiation.artifact import NegotiationCard\n",
"\n",
"report = Report(\n",
" summary=SummaryDescriptor(\n",
" problem_type=ProblemType.CLASSIFICATION, task=\"Iris classification\"\n",
" ),\n",
" performance=PerformanceDesciptor(\n",
" goals=[\n",
" GoalDescriptor(\n",
" description=\"The model should perform well.\",\n",
" metrics=[\n",
" MetricDescriptor(\n",
" description=\"accuracy\",\n",
" baseline=\"Better than random chance.\",\n",
" )\n",
" ],\n",
" )\n",
" ],\n",
" validated_spec_id=validated_spec.identifier,\n",
" ),\n",
" intended_use=IntendedUseDescriptor(\n",
" usage_context=\"A handheld flower identification device.\",\n",
" production_requirements=ModelProductionDescriptor(\n",
" deployment_platform=\"local server\",\n",
" capability_deployment_mechanism=\"API\",\n",
" interface=ModelInterfaceDescriptor(\n",
" input=ModelIODescriptor(\n",
" name=\"i1\", description=\"description\", type=\"Vector[150]\"\n",
" ),\n",
" output=ModelIODescriptor(\n",
" name=\"o1\", description=\"description\", type=\"Vector[3]\"\n",
" ),\n",
" ),\n",
" resources=ModelResourcesDescriptor(\n",
" cpu=\"1\", gpu=\"0\", memory=\"6MiB\", storage=\"2KiB\"\n",
" ),\n",
" ),\n",
" ),\n",
" risks=RiskDescriptor(\n",
" fp=\"The wrong type of flower is identified.\",\n",
" fn=\"The flower is not identified.\",\n",
" other=\"N/A\",\n",
" ),\n",
" data=[\n",
" DataDescriptor(\n",
" description=\"Iris flower dataset.\",\n",
" classification=DataClassification.UNCLASSIFIED,\n",
" access=\"None\",\n",
" labeling_method=\"by hand\",\n",
" fields=[\n",
" FieldDescriptor(\n",
" name=\"Sepal length\",\n",
" description=\"The length of the sepal.\",\n",
" type=\"float\",\n",
" expected_values=\"N/A\",\n",
" missing_values=\"N/A\",\n",
" special_values=\"N/A\",\n",
" )\n",
" ],\n",
" labels=[\n",
" LabelDescriptor(description=\"Setosa\", percentage=30.0),\n",
" LabelDescriptor(description=\"Versicolour\", percentage=30.0),\n",
" LabelDescriptor(description=\"Virginica\", percentage=40.0),\n",
" ],\n",
" policies=\"N/A\",\n",
" rights=\"N/A\",\n",
" source=\"https://archive.ics.uci.edu/dataset/53/iris\",\n",
" )\n",
" ],\n",
" validated_spec_id=validated_spec.identifier,\n",
" comments=[\n",
" CommentDescriptor(\n",
" content=\"This model should not be used for nefarious purposes.\"\n",
Expand All @@ -219,6 +138,9 @@
" ),\n",
")\n",
"\n",
"negotiation_card = NegotiationCard.load()\n",
"report = report.populate_from(negotiation_card)\n",
"\n",
"report.save(force=True, parents=True)"
]
}
Expand Down
Loading

0 comments on commit 55a61be

Please sign in to comment.