Fix defining attributes (#532)

neptune/new/attributes/__init__.py

@@ -19,6 +19,10 @@
 from .atoms.datetime import Datetime
 from .atoms.file import File
 from .atoms.git_ref import GitRef
+from .atoms.boolean import Boolean
+from .atoms.integer import Integer
+from .atoms.run_state import RunState
+from .atoms.notebook_ref import NotebookRef
 
 from .series.float_series import FloatSeries
 from .series.string_series import StringSeries
@@ -27,3 +31,5 @@
 from .sets.string_set import StringSet
 
 from .file_set import FileSet
+
+from .utils import attribute_type_to_atom

neptune/new/attributes/atoms/datetime.py

@@ -29,7 +29,7 @@ def assign(self, value: Union[DatetimeVal, datetime], wait: bool = False):
         if isinstance(value, DatetimeVal):
             value = value.value
         else:
-            value = value.replace(microsecond=1000*int(value.microsecond/1000))
+            value = value.replace(microsecond=1000 * int(value.microsecond / 1000))
         with self._run.lock():
             self._enqueue_operation(AssignDatetime(self._path, value), wait)
 

neptune/new/attributes/atoms/notebook_ref.py

@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2021, Neptune Labs Sp. z o.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from neptune.new.attributes.atoms.atom import Atom
+
+
+class NotebookRef(Atom):
+    pass

neptune/new/attributes/utils.py

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2021, Neptune Labs Sp. z o.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from neptune.new.attributes import (
+    Boolean, Datetime, File, FileSeries, FileSet, Float, FloatSeries, GitRef, Integer, NotebookRef, RunState, String,
+    StringSeries, StringSet,
+)
+from neptune.new.internal.backends.api_model import AttributeType
+
+attribute_type_to_atom = {
+    AttributeType.FLOAT: Float,
+    AttributeType.INT: Integer,
+    AttributeType.BOOL: Boolean,
+    AttributeType.STRING: String,
+    AttributeType.DATETIME: Datetime,
+    AttributeType.FILE: File,
+    AttributeType.FILE_SET: FileSet,
+    AttributeType.FLOAT_SERIES: FloatSeries,
+    AttributeType.STRING_SERIES: StringSeries,
+    AttributeType.IMAGE_SERIES: FileSeries,
+    AttributeType.STRING_SET: StringSet,
+    AttributeType.GIT_REF: GitRef,
+    AttributeType.RUN_STATE: RunState,
+    AttributeType.NOTEBOOK_REF: NotebookRef,
+}

neptune/new/internal/backends/hosted_neptune_backend.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import logging
 import os
 import platform
 import uuid
@@ -93,6 +94,8 @@
 from neptune.new.version import version as neptune_client_version
 from neptune.oauth import NeptuneAuthenticator
 
+_logger = logging.getLogger(__name__)
+
 
 class HostedNeptuneBackend(NeptuneBackend):
     BACKEND_SWAGGER_PATH = "/api/backend/swagger.json"
@@ -324,7 +327,19 @@ def to_attribute(attr) -> Attribute:
         }
         try:
             run = self.leaderboard_client.api.getExperimentAttributes(**params).response().result
-            return [to_attribute(attr) for attr in run.attributes]
+
+            attribute_type_names = [at.value for at in AttributeType]
+            accepted_attributes = [attr for attr in run.attributes if attr.type in attribute_type_names]
+
+            # Notify about ignored attrs
+            ignored_attributes = set(run.attributes) - set(accepted_attributes)
+            if ignored_attributes:
+                _logger.warning(
+                    f"Ignored following attributes (unknown type): {ignored_attributes}.\n"
+                    f"Try to upgrade `neptune-client."
+                )
+
+            return [to_attribute(attr) for attr in accepted_attributes if attr.type in attribute_type_names]
         except HTTPNotFound:
             raise RunUUIDNotFound(run_uuid=run_uuid)
 

neptune/new/run.py

@@ -20,39 +20,29 @@
 import uuid
 from contextlib import AbstractContextManager
 from datetime import datetime
-from typing import Dict, Any, Union, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import click
 
-from neptune.new.attributes.atoms.datetime import Datetime as DatetimeAttr
-from neptune.new.attributes.atoms.run_state import RunState as RunStateAttr
-from neptune.new.attributes.atoms.file import File as FileAttr
-from neptune.new.attributes.atoms.float import Float as FloatAttr
-from neptune.new.attributes.atoms.git_ref import GitRef as GitRefAttr
-from neptune.new.attributes.atoms.string import String as StringAttr
+from neptune.exceptions import UNIX_STYLES
+from neptune.new.attributes import attribute_type_to_atom
 from neptune.new.attributes.attribute import Attribute
-from neptune.new.attributes.file_set import FileSet as FileSetAttr
-from neptune.new.attributes.series.float_series import FloatSeries as FloatSeriesAttr
-from neptune.new.attributes.series.file_series import FileSeries as ImageSeriesAttr
-from neptune.new.attributes.series.string_series import StringSeries as StringSeriesAttr
-from neptune.new.attributes.sets.string_set import StringSet as StringSetAttr
-from neptune.new.exceptions import MetadataInconsistency
+from neptune.new.exceptions import MetadataInconsistency, NeptuneException
 from neptune.new.handler import Handler
 from neptune.new.internal.backends.api_model import AttributeType
 from neptune.new.internal.backends.neptune_backend import NeptuneBackend
 from neptune.new.internal.background_job import BackgroundJob
-from neptune.new.internal.run_structure import RunStructure
 from neptune.new.internal.operation import DeleteAttribute
 from neptune.new.internal.operation_processors.operation_processor import OperationProcessor
-from neptune.new.internal.utils import is_bool, is_int, verify_type, is_float, is_string, is_float_like, is_string_like
+from neptune.new.internal.run_structure import RunStructure
+from neptune.new.internal.utils import is_bool, is_float, is_float_like, is_int, is_string, is_string_like, verify_type
 from neptune.new.internal.utils.paths import parse_path
 from neptune.new.internal.value_to_attribute_visitor import ValueToAttributeVisitor
 from neptune.new.types import Boolean, Integer
 from neptune.new.types.atoms.datetime import Datetime
 from neptune.new.types.atoms.float import Float
 from neptune.new.types.atoms.string import String
 from neptune.new.types.value import Value
-from neptune.exceptions import UNIX_STYLES
 
 
 class Run(AbstractContextManager):
@@ -126,7 +116,7 @@ def _print_structure_impl(self, struct: dict, indent: int) -> None:
                     blue=UNIX_STYLES['blue'],
                     key=key,
                     end=UNIX_STYLES['end']))
-                self._print_structure_impl(struct[key], indent=indent+1)
+                self._print_structure_impl(struct[key], indent=indent + 1)
             else:
                 click.echo("{blue}'{key}'{end}: {type}".format(
                     blue=UNIX_STYLES['blue'],
@@ -207,28 +197,12 @@ def sync(self, wait: bool = True):
                 self._define_attribute(parse_path(attribute.path), attribute.type)
 
     def _define_attribute(self, _path: List[str], _type: AttributeType):
-        if _type == AttributeType.FLOAT:
-            self._structure.set(_path, FloatAttr(self, _path))
-        if _type == AttributeType.STRING:
-            self._structure.set(_path, StringAttr(self, _path))
-        if _type == AttributeType.DATETIME:
-            self._structure.set(_path, DatetimeAttr(self, _path))
-        if _type == AttributeType.FILE:
-            self._structure.set(_path, FileAttr(self, _path))
-        if _type == AttributeType.FILE_SET:
-            self._structure.set(_path, FileSetAttr(self, _path))
-        if _type == AttributeType.FLOAT_SERIES:
-            self._structure.set(_path, FloatSeriesAttr(self, _path))
-        if _type == AttributeType.STRING_SERIES:
-            self._structure.set(_path, StringSeriesAttr(self, _path))
-        if _type == AttributeType.IMAGE_SERIES:
-            self._structure.set(_path, ImageSeriesAttr(self, _path))
-        if _type == AttributeType.STRING_SET:
-            self._structure.set(_path, StringSetAttr(self, _path))
-        if _type == AttributeType.GIT_REF:
-            self._structure.set(_path, GitRefAttr(self, _path))
-        if _type == AttributeType.RUN_STATE:
-            self._structure.set(_path, RunStateAttr(self, _path))
+        try:
+            attr_init = attribute_type_to_atom[_type]
+        except KeyError:
+            raise NeptuneException(f"Unexpected type: {_type}")
+
+        self._structure.set(_path, attr_init(self, _path))
 
     def _shutdown_hook(self):
         self.stop()

tests/neptune/new/attributes/test_attribute_utils.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2021, Neptune Labs Sp. z o.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+from neptune.new.attributes import attribute_type_to_atom
+from neptune.new.attributes.attribute import Attribute
+from neptune.new.internal.backends.api_model import AttributeType
+
+
+class TestAttributeUtils(unittest.TestCase):
+    def test_attribute_type_to_atom(self):
+        # Expect all AttributeTypes are reflected in `attribute_type_to_atom`...
+        self.assertEqual(set(at for at in AttributeType), set(attribute_type_to_atom.keys()))
+
+        # ... and this reflection is class based on `Attribute`
+        self.assertTrue(all(issubclass(attr_atom, Attribute) for attr_atom in attribute_type_to_atom.values()))

tests/neptune/new/test_run.py

@@ -15,14 +15,14 @@
 #
 import os
 import unittest
+from datetime import datetime
 
-from neptune.new.types.series import StringSeries, FloatSeries
-
-from neptune.new import init, ANONYMOUS
-from neptune.new.envs import PROJECT_ENV_NAME, API_TOKEN_ENV_NAME
+from neptune.new import ANONYMOUS, init
+from neptune.new.envs import API_TOKEN_ENV_NAME, PROJECT_ENV_NAME
 from neptune.new.exceptions import MetadataInconsistency
 from neptune.new.types.atoms.float import Float
 from neptune.new.types.atoms.string import String
+from neptune.new.types.series import FloatSeries, StringSeries
 
 
 class TestRun(unittest.TestCase):
@@ -74,6 +74,7 @@ def test_run_as_handler(self):
 
     def test_assign_dict(self):
         exp = init(mode="debug", flush_period=0.5)
+        now = datetime.now()
         exp.assign({
             "x": 5,
             "metadata": {
@@ -85,10 +86,23 @@ def test_assign_dict(self):
                 "nested": {
                     "deep_secret": FloatSeries([13, 15])
                 }
+            },
+            "simple_types": {
+                "int": 42,
+                "str": "imagine",
+                "float": 3.14,
+                "datetime": now,
+                "list": list(range(10)),
             }
         })
         self.assertEqual(exp['x'].fetch(), 5)
         self.assertEqual(exp['metadata/name'].fetch(), "Trol")
         self.assertEqual(exp['metadata/age'].fetch(), 376)
         self.assertEqual(exp['toys'].fetch_last(), "hat")
         self.assertEqual(exp['nested/nested/deep_secret'].fetch_last(), 15)
+        self.assertEqual(exp['simple_types/int'].fetch(), 42)
+        self.assertEqual(exp['simple_types/str'].fetch(), 'imagine')
+        self.assertEqual(exp['simple_types/float'].fetch(), 3.14)
+        self.assertEqual(exp['simple_types/datetime'].fetch(),
+                         now.replace(microsecond=1000 * int(now.microsecond / 1000)))
+        self.assertEqual(exp['simple_types/list'].fetch(), str(list(range(10))))