diff --git a/demo/scenarios/report.ipynb b/demo/scenarios/report.ipynb index 8c1f421a9..45999ca22 100644 --- a/demo/scenarios/report.ipynb +++ b/demo/scenarios/report.ipynb @@ -76,35 +76,14 @@ "source": [ "from mlte.spec.spec import Spec\n", "from mlte.validation.spec_validator import SpecValidator\n", - "\n", - "from mlte.measurement.cpu import CPUStatistics\n", - "from mlte.measurement.memory import MemoryStatistics\n", - "from mlte.value.types.image import Image\n", - "from mlte.value.types.integer import Integer\n", - "\n", - "from values.multiple_accuracy import MultipleAccuracy\n", - "from values.multiple_ranksums import MultipleRanksums\n", - "from values.ranksums import RankSums\n", + "from mlte.value.artifact import Value\n", "\n", "# Load the specification\n", "spec = Spec.load()\n", "\n", "# Add all values to the validator.\n", "spec_validator = SpecValidator(spec)\n", - "spec_validator.add_value(MultipleAccuracy.load(\"accuracy across gardens.value\"))\n", - "spec_validator.add_value(RankSums.load(\"ranksums blur2x8.value\"))\n", - "spec_validator.add_value(RankSums.load(\"ranksums blur5x8.value\"))\n", - "spec_validator.add_value(RankSums.load(\"ranksums blur0x8.value\"))\n", - "spec_validator.add_value(\n", - " MultipleRanksums.load(\"multiple ranksums for clade2.value\")\n", - ")\n", - "spec_validator.add_value(\n", - " MultipleRanksums.load(\"multiple ranksums between clade2 and 3.value\")\n", - ")\n", - "spec_validator.add_value(Integer.load(\"model size.value\"))\n", - "spec_validator.add_value(CPUStatistics.load(\"predicting cpu.value\"))\n", - "spec_validator.add_value(MemoryStatistics.load(\"predicting memory.value\"))\n", - "spec_validator.add_value(Image.load(\"image attributions.value\"))" + "spec_validator.add_values(Value.load_all())" ] }, { diff --git a/demo/simple/report.ipynb b/demo/simple/report.ipynb index e92c09632..7d3486f53 100644 --- a/demo/simple/report.ipynb +++ b/demo/simple/report.ipynb @@ -80,26 +80,14 @@ "source": [ "from mlte.spec.spec import Spec\n", "from mlte.validation.spec_validator import SpecValidator\n", - "from mlte.value.types.integer import Integer\n", - "from mlte.value.types.real import Real\n", - "from mlte.value.types.image import Image\n", - "from mlte.measurement.cpu import CPUStatistics\n", - "from mlte.measurement.memory import MemoryStatistics\n", - "from confusion_matrix import ConfusionMatrix\n", + "from mlte.value.artifact import Value\n", "\n", "# Load the specification\n", "spec = Spec.load()\n", "\n", - "# TODO: This could also be done in bulk, with a dictionary-binding like indicating properties, validators and ids.\n", - "\n", "# Add all values to the validator.\n", "spec_validator = SpecValidator(spec)\n", - "spec_validator.add_value(Integer.load(\"model size.value\"))\n", - "spec_validator.add_value(CPUStatistics.load(\"training cpu.value\"))\n", - "spec_validator.add_value(MemoryStatistics.load(\"training memory.value\"))\n", - "spec_validator.add_value(Real.load(\"accuracy.value\"))\n", - "spec_validator.add_value(ConfusionMatrix.load(\"confusion matrix.value\"))\n", - "spec_validator.add_value(Image.load(\"class distribution.value\"))" + "spec_validator.add_values(Value.load_all())" ] }, { diff --git a/mlte/_private/reflection.py b/mlte/_private/reflection.py new file mode 100644 index 000000000..81b8d6f95 --- /dev/null +++ b/mlte/_private/reflection.py @@ -0,0 +1,26 @@ +import importlib +from typing import Any, Type + + +def load_class(class_path: str) -> Type[Any]: + """ + Returns a class type of the given class name/path. + :param class_path: A path to a class to use, including absolute package/module path and class name. + """ + # Split into package/module and class name. + parts = class_path.rsplit(".", 1) + module_name = parts[0] + class_name = parts[1] + + try: + loaded_module = importlib.import_module(module_name) + except Exception: + raise RuntimeError(f"Module {module_name} not found") + try: + class_type: Type[Any] = getattr(loaded_module, class_name) + except Exception: + raise RuntimeError( + f"Class {class_name} in module {module_name} not found" + ) + + return class_type diff --git a/mlte/artifact/artifact.py b/mlte/artifact/artifact.py index 07706179f..e6ba20e0b 100644 --- a/mlte/artifact/artifact.py +++ b/mlte/artifact/artifact.py @@ -11,6 +11,7 @@ from typing import Optional import mlte._private.meta as meta +import mlte.store.query as query from mlte.artifact.model import ArtifactHeaderModel, ArtifactModel from mlte.artifact.type import ArtifactType from mlte.context.context import Context @@ -166,6 +167,32 @@ def load_with( artifact.post_load_hook(context, store) return artifact + @staticmethod + def load_all_models(artifact_type: ArtifactType) -> list[ArtifactModel]: + """Loads all artifact models of the given type from the session.""" + return Artifact.load_all_models_with( + artifact_type, context=session().context, store=session().store + ) + + @staticmethod + def load_all_models_with( + artifact_type: ArtifactType, context: Context, store: Store + ) -> list[ArtifactModel]: + """Loads all artifact models of the given type for the given context and store.""" + with ManagedSession(store.session()) as handle: + query_instance = query.Query( + filter=query.ArtifactTypeFilter( + type=query.FilterType.TYPE, artifact_type=artifact_type + ) + ) + artifact_models = handle.search_artifacts( + context.namespace, + context.model, + context.version, + query_instance, + ) + return artifact_models + @staticmethod def get_default_id() -> str: """To be overriden by derived classes.""" diff --git a/mlte/frontend/nuxt-app/assets/schema/artifact/value/v0.0.1/schema.json b/mlte/frontend/nuxt-app/assets/schema/artifact/value/v0.0.1/schema.json index 7a2ae9b33..5939e2199 100644 --- a/mlte/frontend/nuxt-app/assets/schema/artifact/value/v0.0.1/schema.json +++ b/mlte/frontend/nuxt-app/assets/schema/artifact/value/v0.0.1/schema.json @@ -130,6 +130,10 @@ "metadata": { "$ref": "#/$defs/EvidenceMetadata" }, + "value_class": { + "title": "Value Class", + "type": "string" + }, "value": { "discriminator": { "mapping": { @@ -160,6 +164,7 @@ "required": [ "artifact_type", "metadata", + "value_class", "value" ], "title": "ValueModel", diff --git a/mlte/schema/artifact/value/v0.0.1/schema.json b/mlte/schema/artifact/value/v0.0.1/schema.json index 7a2ae9b33..5939e2199 100644 --- a/mlte/schema/artifact/value/v0.0.1/schema.json +++ b/mlte/schema/artifact/value/v0.0.1/schema.json @@ -130,6 +130,10 @@ "metadata": { "$ref": "#/$defs/EvidenceMetadata" }, + "value_class": { + "title": "Value Class", + "type": "string" + }, "value": { "discriminator": { "mapping": { @@ -160,6 +164,7 @@ "required": [ "artifact_type", "metadata", + "value_class", "value" ], "title": "ValueModel", diff --git a/mlte/value/artifact.py b/mlte/value/artifact.py index a711783c5..0061b57f7 100644 --- a/mlte/value/artifact.py +++ b/mlte/value/artifact.py @@ -7,11 +7,16 @@ from __future__ import annotations import abc +import typing +from mlte._private.reflection import load_class from mlte.artifact.artifact import Artifact from mlte.artifact.model import ArtifactModel from mlte.artifact.type import ArtifactType +from mlte.context.context import Context from mlte.evidence.metadata import EvidenceMetadata +from mlte.store.base import Store +from mlte.value.model import ValueModel class Value(Artifact, metaclass=abc.ABCMeta): @@ -57,3 +62,37 @@ def from_model(cls, _: ArtifactModel) -> Value: # type: ignore[override] is delegated to subclasses that implement concrete types """ raise NotImplementedError("Value.from_model()") + + @staticmethod + def load_all() -> list[Value]: + """Loads all artifact models of the given type for the current session.""" + value_models = Value.load_all_models(ArtifactType.VALUE) + return Value._load_from_models(value_models) + + @staticmethod + def load_all_with(context: Context, store: Store) -> list[Value]: + """Loads all artifact models of the given type for the given context and store.""" + value_models = Value.load_all_models_with( + ArtifactType.VALUE, context, store + ) + return Value._load_from_models(value_models) + + @staticmethod + def _load_from_models(value_models: list[ArtifactModel]) -> list[Value]: + """Converts a list of value models (as Artifact Models) into values.""" + values = [] + for artifact_model in value_models: + value_model: ValueModel = typing.cast( + ValueModel, artifact_model.body + ) + value_type: Value = typing.cast( + Value, load_class(value_model.value_class) + ) + value = value_type.from_model(artifact_model) + values.append(value) + return values + + @classmethod + def get_class_path(cls) -> str: + """Returns the full path to this class, including module.""" + return f"{cls.__module__}.{cls.__name__}" diff --git a/mlte/value/base.py b/mlte/value/base.py index ee3c9b316..37f035f84 100644 --- a/mlte/value/base.py +++ b/mlte/value/base.py @@ -33,14 +33,6 @@ def __subclasshook__(cls, subclass): """Define the interface for all Value subclasses.""" return meta.has_callables(subclass, "serialize", "deserialize") - def __init__(self, instance: ValueBase, metadata: EvidenceMetadata) -> None: - """ - Initialize a MLTE value. - :param instance: The subclass instance - :param metadata: Evidence metadata associated with the value - """ - super().__init__(instance, metadata) - @abc.abstractmethod def serialize(self) -> Dict[str, Any]: """ @@ -72,8 +64,10 @@ def to_model(self) -> ArtifactModel: body=ValueModel( artifact_type=ArtifactType.VALUE, metadata=self.metadata, + value_class=self.get_class_path(), value=OpaqueValueModel( - value_type=ValueType.OPAQUE, data=self.serialize() + value_type=ValueType.OPAQUE, + data=self.serialize(), ), ), ) diff --git a/mlte/value/model.py b/mlte/value/model.py index 45451c9a8..eaa32a185 100644 --- a/mlte/value/model.py +++ b/mlte/value/model.py @@ -41,6 +41,9 @@ class ValueModel(BaseModel): metadata: EvidenceMetadata """Evidence metadata associated with the value.""" + value_class: str + """Full path to class that implements this value.""" + value: Union[ "IntegerValueModel", "RealValueModel", diff --git a/mlte/value/types/image.py b/mlte/value/types/image.py index 85770e462..7225a77cb 100644 --- a/mlte/value/types/image.py +++ b/mlte/value/types/image.py @@ -61,6 +61,7 @@ def to_model(self) -> ArtifactModel: body=ValueModel( artifact_type=ArtifactType.VALUE, metadata=self.metadata, + value_class=self.get_class_path(), value=ImageValueModel( value_type=ValueType.IMAGE, data=base64.encodebytes(self.image).decode("utf-8"), diff --git a/mlte/value/types/integer.py b/mlte/value/types/integer.py index 06eb0cb70..6bde2a056 100644 --- a/mlte/value/types/integer.py +++ b/mlte/value/types/integer.py @@ -44,8 +44,10 @@ def to_model(self) -> ArtifactModel: body=ValueModel( artifact_type=ArtifactType.VALUE, metadata=self.metadata, + value_class=self.get_class_path(), value=IntegerValueModel( - value_type=ValueType.INTEGER, integer=self.value + value_type=ValueType.INTEGER, + integer=self.value, ), ), ) diff --git a/mlte/value/types/opaque.py b/mlte/value/types/opaque.py index 66d5bfd92..5654a99aa 100644 --- a/mlte/value/types/opaque.py +++ b/mlte/value/types/opaque.py @@ -44,8 +44,10 @@ def to_model(self) -> ArtifactModel: body=ValueModel( artifact_type=ArtifactType.VALUE, metadata=self.metadata, + value_class=self.get_class_path(), value=OpaqueValueModel( - value_type=ValueType.OPAQUE, data=self.data + value_type=ValueType.OPAQUE, + data=self.data, ), ), ) diff --git a/mlte/value/types/real.py b/mlte/value/types/real.py index c44bee963..14f626812 100644 --- a/mlte/value/types/real.py +++ b/mlte/value/types/real.py @@ -45,8 +45,10 @@ def to_model(self) -> ArtifactModel: body=ValueModel( artifact_type=ArtifactType.VALUE, metadata=self.metadata, + value_class=self.get_class_path(), value=RealValueModel( - value_type=ValueType.REAL, real=self.value + value_type=ValueType.REAL, + real=self.value, ), ), ) diff --git a/test/artifact/test_artifact.py b/test/artifact/test_artifact.py index e2776c728..c7727bb86 100644 --- a/test/artifact/test_artifact.py +++ b/test/artifact/test_artifact.py @@ -3,10 +3,24 @@ Unit tests for MLTE artifact protocol implementation. """ +from typing import Tuple +import pytest + +from mlte.artifact.artifact import Artifact +from mlte.artifact.type import ArtifactType +from mlte.context.context import Context +from mlte.evidence.metadata import EvidenceMetadata, Identifier from mlte.negotiation.artifact import NegotiationCard +from mlte.report.artifact import Report from mlte.session.state import set_context, set_store +from mlte.spec.spec import Spec +from mlte.store.base import Store +from mlte.validation.validated_spec import ValidatedSpec +from mlte.value.types.integer import Integer +from mlte.value.types.real import Real +from ..fixture.store import store_with_context # noqa from ..fixture.store import FX_MODEL_ID, FX_NAMESPACE_ID, FX_VERSION_ID @@ -24,3 +38,51 @@ def test_save_load_session() -> None: a.save(parents=True) _ = NegotiationCard.load("my-card") + + +def fill_test_store(ctx: Context, store: Store): + """Fills a sample store.""" + n1 = NegotiationCard("test-card") + n2 = NegotiationCard("test-card2") + s1 = Spec("test-spec1") + s2 = Spec("test-spec2") + vs1 = ValidatedSpec("test-validated1", s1) + vs2 = ValidatedSpec("test-validated2", s2) + m1 = EvidenceMetadata( + measurement_type="typename", identifier=Identifier(name="id1") + ) + v1 = Integer(m1, 10) + m2 = EvidenceMetadata( + measurement_type="typename", identifier=Identifier(name="id2") + ) + v2 = Real(m2, 3.14) + r1 = Report("r1") + r2 = Report("r2") + + n1.save_with(ctx, store, parents=True) + n2.save_with(ctx, store) + s1.save_with(ctx, store) + s2.save_with(ctx, store) + vs1.save_with(ctx, store) + vs2.save_with(ctx, store) + v1.save_with(ctx, store) + v2.save_with(ctx, store) + r1.save_with(ctx, store) + r2.save_with(ctx, store) + + +@pytest.mark.parametrize("artifact_type", ArtifactType) +def test_load_all_models( + artifact_type: ArtifactType, + store_with_context: Tuple[Store, Context], # noqa +): + """ + Loading all models of a given type. + """ + store, ctx = store_with_context + fill_test_store(ctx, store) + + models = Artifact.load_all_models_with(artifact_type, ctx, store) + + assert len(models) == 2 + assert models[0].header.type == artifact_type diff --git a/test/fixture/artifact.py b/test/fixture/artifact.py index 80c21d977..bac4e4100 100644 --- a/test/fixture/artifact.py +++ b/test/fixture/artifact.py @@ -18,6 +18,7 @@ from mlte.spec.model import SpecModel from mlte.validation.model import ValidatedSpecModel from mlte.value.model import IntegerValueModel, ValueModel, ValueType +from mlte.value.types.integer import Integer def _random_id(length: int = 5) -> str: @@ -109,7 +110,11 @@ def _make_value(id: str) -> ValueModel: return ValueModel( artifact_type=ArtifactType.VALUE, metadata=m, - value=IntegerValueModel(value_type=ValueType.INTEGER, integer=1), + value_class=Integer.get_class_path(), + value=IntegerValueModel( + value_type=ValueType.INTEGER, + integer=1, + ), ) diff --git a/test/value/test_value.py b/test/value/test_value.py new file mode 100644 index 000000000..af6f4a915 --- /dev/null +++ b/test/value/test_value.py @@ -0,0 +1,42 @@ +""" +mlte/value/test_value.py + +Unit tests for Value. +""" + +from typing import Tuple + +from mlte.context.context import Context +from mlte.evidence.metadata import EvidenceMetadata, Identifier +from mlte.store.base import Store +from mlte.value.artifact import Value +from mlte.value.types.integer import Integer +from mlte.value.types.real import Real + +from ..fixture.store import store_with_context # noqa + + +def test_load_all( + store_with_context: Tuple[Store, Context], # noqa +): + """ + Loading all models of a given type. + """ + store, ctx = store_with_context + + m1 = EvidenceMetadata( + measurement_type="typename", identifier=Identifier(name="id1") + ) + v1 = Integer(m1, 10) + m2 = EvidenceMetadata( + measurement_type="typename", identifier=Identifier(name="id2") + ) + v2 = Real(m2, 3.14) + v1.save_with(ctx, store, parents=True) + v2.save_with(ctx, store) + + models = Value.load_all_with(ctx, store) + + assert len(models) == 2 + assert models[0] == v1 or models[0] == v2 + assert models[1] == v1 or models[1] == v2