update limits and limit warning messages (#561)

* collect all limits in single file
* update upload limits and error messages

neptune/new/internal/utils/__init__.py

@@ -20,6 +20,8 @@
 from glob import glob
 from typing import Union, TypeVar, Iterable, List, Set, Optional, Mapping
 
+from neptune.new.internal.utils import limits
+
 T = TypeVar('T')
 
 _logger = logging.getLogger(__name__)
@@ -127,22 +129,16 @@ def get_common_root(absolute_paths: List[str]) -> Optional[str]:
         return None
 
 
-STREAM_SIZE_LIMIT_MB = 15
-
-
 def get_stream_content(stream: IOBase, seek: Optional[int] = None) -> (Optional[str], str):
     if seek is not None and stream.seekable():
         stream.seek(seek)
 
-    content = stream.read(STREAM_SIZE_LIMIT_MB * 1024 * 1024 + 1)
+    content = stream.read(limits.STREAM_SIZE_LIMIT_BYTES + 1)
     default_ext = "txt" if isinstance(content, str) else "bin"
     if isinstance(content, str):
         content = content.encode('utf-8')
 
-    if len(content) > STREAM_SIZE_LIMIT_MB * 1024 * 1024:
-        _logger.warning('Your stream is larger than %dMB. '
-                        'Neptune supports saving files from streams smaller than %dMB.',
-                        STREAM_SIZE_LIMIT_MB, STREAM_SIZE_LIMIT_MB)
+    if limits.stream_size_exceeds_limit(len(content)):
         return None, default_ext
 
     while True:
@@ -157,9 +153,7 @@ def get_stream_content(stream: IOBase, seek: Optional[int] = None) -> (Optional[
                 content.seek(0, 2)
             if isinstance(chunk, str):
                 chunk = chunk.encode('utf-8')
-            if content.tell() + len(chunk) > STREAM_SIZE_LIMIT_MB * 1024 * 1024:
-                _logger.warning('Your stream is larger than %dMB. Neptune supports saving files smaller than %dMB.',
-                                STREAM_SIZE_LIMIT_MB, STREAM_SIZE_LIMIT_MB)
+            if limits.stream_size_exceeds_limit(content.tell() + len(chunk)):
                 return None, default_ext
             content.write(chunk)
     if isinstance(content, BytesIO):

neptune/new/internal/utils/images.py

@@ -25,6 +25,7 @@
 from pandas import DataFrame
 
 from neptune.new.exceptions import PlotlyIncompatibilityException
+from neptune.new.internal.utils import limits
 
 _logger = logging.getLogger(__name__)
 
@@ -44,17 +45,10 @@ def pilimage_fromarray():
         pass
 
 
-IMAGE_SIZE_LIMIT_MB = 15
-
-
 def get_image_content(image) -> Optional[bytes]:
     content = _image_to_bytes(image)
 
-    if len(content) > IMAGE_SIZE_LIMIT_MB * 1024 * 1024:
-        _logger.warning('Your image is larger than %dMB. Neptune supports logging images smaller than %dMB. '
-                        'Resize or increase compression of this image',
-                        IMAGE_SIZE_LIMIT_MB,
-                        IMAGE_SIZE_LIMIT_MB)
+    if limits.image_size_exceeds_limit(len(content)):
         return None
 
     return content
@@ -63,12 +57,7 @@ def get_image_content(image) -> Optional[bytes]:
 def get_html_content(chart) -> Optional[str]:
     content = _to_html(chart)
 
-    if len(content) > IMAGE_SIZE_LIMIT_MB * 1024 * 1024:
-        _logger.warning('Your file is larger than %dMB. '
-                        'Neptune supports logging files in-memory objects smaller than %dMB. '
-                        'Resize or increase compression of this object',
-                        IMAGE_SIZE_LIMIT_MB,
-                        IMAGE_SIZE_LIMIT_MB)
+    if limits.file_size_exceeds_limit(len(content)):
         return None
 
     return content
@@ -77,12 +66,7 @@ def get_html_content(chart) -> Optional[str]:
 def get_pickle_content(obj) -> Optional[bytes]:
     content = _export_pickle(obj)
 
-    if len(content) > IMAGE_SIZE_LIMIT_MB * 1024 * 1024:
-        _logger.warning('Your file is larger than %dMB. '
-                        'Neptune supports logging files in-memory objects smaller than %dMB. '
-                        'Resize or increase compression of this object',
-                        IMAGE_SIZE_LIMIT_MB,
-                        IMAGE_SIZE_LIMIT_MB)
+    if limits.file_size_exceeds_limit(len(content)):
         return None
 
     return content

neptune/new/internal/utils/limits.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2021, Neptune Labs Sp. z o.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+
+_logger = logging.getLogger(__name__)
+
+
+_IMAGE_SIZE_LIMIT_MB = 15
+_IN_MEMORY_SIZE_LIMIT_MB = 32
+_STREAM_SIZE_LIMIT_MB = 32
+
+BYTES_IN_MB = 1024 * 1024
+
+STREAM_SIZE_LIMIT_BYTES = _STREAM_SIZE_LIMIT_MB * BYTES_IN_MB
+
+
+def image_size_exceeds_limit(content_size):
+    if content_size > _IMAGE_SIZE_LIMIT_MB * BYTES_IN_MB:
+        _logger.warning('You are attempting to create an image that is %.2fMB large. '
+                        'Neptune supports logging images smaller than %dMB. '
+                        'Resize or increase compression of this image',
+                        content_size / BYTES_IN_MB,
+                        _IMAGE_SIZE_LIMIT_MB)
+        return True
+    return False
+
+
+def file_size_exceeds_limit(content_size):
+    if content_size > _IN_MEMORY_SIZE_LIMIT_MB * BYTES_IN_MB:
+        _logger.warning('You are attempting to create an in-memory file that is %.1fMB large. '
+                        'Neptune supports logging in-memory file objects smaller than %dMB. '
+                        'Resize or increase compression of this object',
+                        content_size / BYTES_IN_MB,
+                        _IN_MEMORY_SIZE_LIMIT_MB)
+        return True
+    return False
+
+
+def stream_size_exceeds_limit(content_size):
+    if content_size > _STREAM_SIZE_LIMIT_MB * BYTES_IN_MB:
+        _logger.warning('Your stream is larger than %dMB. '
+                        'Neptune supports saving files from streams smaller than %dMB.',
+                        _STREAM_SIZE_LIMIT_MB,
+                        _STREAM_SIZE_LIMIT_MB)
+        return True
+    return False

tests/neptune/new/internal/utils/test_images.py

@@ -20,6 +20,7 @@
 import sys
 import unittest
 from typing import Optional
+from unittest import mock
 from uuid import uuid4
 
 import matplotlib
@@ -198,6 +199,25 @@ def test_get_html_from_pandas(self):
         self.assertTrue(result.startswith(
             '<table border="1" class="dataframe">\n  <thead>\n    <tr style="text-align: right;">'))
 
+    def test_get_oversize_html_from_pandas(self):
+        # given
+        table = mock.Mock(spec=pandas.DataFrame)
+        table.to_html.return_value = 40_000_000 * 'a'
+
+        # when
+        with self.assertLogs() as caplog:
+            result = get_html_content(table)
+
+        # then
+        self.assertIsNone(result)
+        self.assertEqual(
+            caplog.output, [
+                'WARNING:neptune.new.internal.utils.limits:You are attempting to create an in-memory file that'
+                ' is 38.1MB large. Neptune supports logging in-memory file objects smaller than 32MB. '
+                'Resize or increase compression of this object'
+            ]
+        )
+
     @staticmethod
     def _encode_pil_image(image: Image) -> bytes:
         with io.BytesIO() as image_buffer: