From 764ee1f065f5720f4ae8dec0187562f891ae5bb3 Mon Sep 17 00:00:00 2001 From: Pawel Date: Mon, 15 Apr 2024 23:01:49 -0700 Subject: [PATCH 1/9] traverse dict for recursive shortening --- rollbar/lib/transforms/shortener.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index f0392912..8685429d 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -77,11 +77,22 @@ def _shorten_other(self, obj): return self._repr.repr(obj) + def traverse_dict(self, d): + max_size = self._get_max_size(d) + d = self._shorten_mapping(d, max_size) + for k, v in d.items(): + if isinstance(v, dict): + self.traverse_dict(v) + else: + d[k] = self._shorten(v) + return d + def _shorten(self, val): max_size = self._get_max_size(val) if isinstance(val, dict): - return self._shorten_mapping(val, max_size) + return self.traverse_dict(val) + if isinstance(val, (string_types, sequence_types)): return self._shorten_sequence(val, max_size) From 97c2a32869b84061bc1a933b1639cdc9c19d1865 Mon Sep 17 00:00:00 2001 From: Pawel Date: Mon, 15 Apr 2024 23:21:28 -0700 Subject: [PATCH 2/9] correct algo --- rollbar/lib/transforms/shortener.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index 8685429d..ae8d810b 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -82,7 +82,9 @@ def traverse_dict(self, d): d = self._shorten_mapping(d, max_size) for k, v in d.items(): if isinstance(v, dict): - self.traverse_dict(v) + max_size = self._get_max_size(v) + d[k] = self._shorten_mapping(v, max_size) + self.traverse_dict(d[k]) else: d[k] = self._shorten(v) return d From ded614b98682f9b6061f0c402967a1404133f030 Mon Sep 17 00:00:00 2001 From: Pawel Date: Mon, 15 Apr 2024 23:25:39 -0700 Subject: [PATCH 3/9] simplify algo --- rollbar/lib/transforms/shortener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index ae8d810b..89ae5a58 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -78,8 +78,6 @@ def _shorten_other(self, obj): return self._repr.repr(obj) def traverse_dict(self, d): - max_size = self._get_max_size(d) - d = self._shorten_mapping(d, max_size) for k, v in d.items(): if isinstance(v, dict): max_size = self._get_max_size(v) @@ -93,6 +91,8 @@ def _shorten(self, val): max_size = self._get_max_size(val) if isinstance(val, dict): + max_size = self._get_max_size(val) + val = self._shorten_mapping(val, max_size) return self.traverse_dict(val) if isinstance(val, (string_types, sequence_types)): From 0dcad823c14d053d04f6f18cdab60358714b1e97 Mon Sep 17 00:00:00 2001 From: Pawel Date: Tue, 16 Apr 2024 01:15:35 -0700 Subject: [PATCH 4/9] traverse object --- rollbar/lib/transforms/shortener.py | 42 +++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index 89ae5a58..e192c186 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -77,27 +77,47 @@ def _shorten_other(self, obj): return self._repr.repr(obj) - def traverse_dict(self, d): - for k, v in d.items(): - if isinstance(v, dict): - max_size = self._get_max_size(v) - d[k] = self._shorten_mapping(v, max_size) - self.traverse_dict(d[k]) - else: - d[k] = self._shorten(v) + def traverse_obj(self, d): + if isinstance(d, dict): + for k, v in d.items(): + if isinstance(v, dict): + max_size = self._get_max_size(v) + d[k] = self._shorten_mapping(v, max_size) + self.traverse_obj(d[k]) + elif isinstance(v, sequence_types): + max_size = self._get_max_size(v) + d[k] = self._shorten_sequence(v, max_size) + self.traverse_obj(d[k]) + else: + d[k] = self._shorten(v) + elif isinstance(d, sequence_types): + for i in range(len(d)): + if isinstance(d[i], dict): + max_size = self._get_max_size(d[i]) + d[i] = self._shorten_mapping(d[i], max_size) + self.traverse_obj(d[i]) + elif isinstance(d[i], sequence_types): + max_size = self._get_max_size(d[i]) + d[i] = self._shorten_sequence(d[i], max_size) + self.traverse_obj(d[i]) + else: + d[i] = self._shorten(d[i]) return d def _shorten(self, val): max_size = self._get_max_size(val) if isinstance(val, dict): - max_size = self._get_max_size(val) val = self._shorten_mapping(val, max_size) - return self.traverse_dict(val) + return self.traverse_obj(val) - if isinstance(val, (string_types, sequence_types)): + if isinstance(val, string_types): return self._shorten_sequence(val, max_size) + if isinstance(val, sequence_types): + val = self._shorten_sequence(val, max_size) + return self.traverse_obj(val) + if isinstance(val, number_types): return self._shorten_basic(val, self._repr.maxlong) From eda09e5543a888aa5ebc3ccb42fe9608444b1363 Mon Sep 17 00:00:00 2001 From: Pawel Date: Tue, 16 Apr 2024 01:26:39 -0700 Subject: [PATCH 5/9] final version --- rollbar/lib/transforms/shortener.py | 41 +++++++++++------------------ 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index e192c186..227dae34 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -77,32 +77,21 @@ def _shorten_other(self, obj): return self._repr.repr(obj) - def traverse_obj(self, d): - if isinstance(d, dict): - for k, v in d.items(): - if isinstance(v, dict): - max_size = self._get_max_size(v) - d[k] = self._shorten_mapping(v, max_size) - self.traverse_obj(d[k]) - elif isinstance(v, sequence_types): - max_size = self._get_max_size(v) - d[k] = self._shorten_sequence(v, max_size) - self.traverse_obj(d[k]) - else: - d[k] = self._shorten(v) - elif isinstance(d, sequence_types): - for i in range(len(d)): - if isinstance(d[i], dict): - max_size = self._get_max_size(d[i]) - d[i] = self._shorten_mapping(d[i], max_size) - self.traverse_obj(d[i]) - elif isinstance(d[i], sequence_types): - max_size = self._get_max_size(d[i]) - d[i] = self._shorten_sequence(d[i], max_size) - self.traverse_obj(d[i]) - else: - d[i] = self._shorten(d[i]) - return d + def traverse_obj(self, obj): + def seq_iter(o): + return o if isinstance(o, dict) else range(len(o)) + + for k in seq_iter(obj): + max_size = self._get_max_size(obj[k]) + if isinstance(obj[k], dict): + obj[k] = self._shorten_mapping(obj[k], max_size) + self.traverse_obj(obj[k]) + elif isinstance(obj[k], sequence_types): + obj[k] = self._shorten_sequence(obj[k], max_size) + self.traverse_obj(obj[k]) + else: + obj[k] = self._shorten(obj[k]) + return obj def _shorten(self, val): max_size = self._get_max_size(val) From d79b19b764335302491aeee6ab81f06f82c32657 Mon Sep 17 00:00:00 2001 From: Pawel Date: Tue, 16 Apr 2024 01:51:03 -0700 Subject: [PATCH 6/9] correct unit tests --- rollbar/lib/transforms/shortener.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index 227dae34..ff83738e 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -105,6 +105,8 @@ def _shorten(self, val): if isinstance(val, sequence_types): val = self._shorten_sequence(val, max_size) + if isinstance(val, string_types): + return val return self.traverse_obj(val) if isinstance(val, number_types): From a32094bd0c19b96795afb63d319108ef02b0930f Mon Sep 17 00:00:00 2001 From: Pawel Date: Wed, 17 Apr 2024 00:20:32 -0700 Subject: [PATCH 7/9] added support for max_level --- rollbar/lib/transforms/shortener.py | 14 +++++++++++--- rollbar/test/test_shortener_transform.py | 16 ++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index ff83738e..d3040a35 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -47,6 +47,8 @@ def _get_max_size(self, obj): return self._repr.maxother + def _get_max_level(self): + return getattr(self._repr, 'maxlevel') def _shorten_sequence(self, obj, max_keys): _len = len(obj) if _len <= max_keys: @@ -77,7 +79,7 @@ def _shorten_other(self, obj): return self._repr.repr(obj) - def traverse_obj(self, obj): + def traverse_obj(self, obj, level=1): def seq_iter(o): return o if isinstance(o, dict) else range(len(o)) @@ -85,10 +87,16 @@ def seq_iter(o): max_size = self._get_max_size(obj[k]) if isinstance(obj[k], dict): obj[k] = self._shorten_mapping(obj[k], max_size) - self.traverse_obj(obj[k]) + if level == self._get_max_level(): + del obj[k] + return + self.traverse_obj(obj[k], level + 1) elif isinstance(obj[k], sequence_types): obj[k] = self._shorten_sequence(obj[k], max_size) - self.traverse_obj(obj[k]) + if level == self._get_max_level(): + del obj[k] + return + self.traverse_obj(obj[k], level + 1) else: obj[k] = self._shorten(obj[k]) return obj diff --git a/rollbar/test/test_shortener_transform.py b/rollbar/test/test_shortener_transform.py index 55180c34..3bcc458c 100644 --- a/rollbar/test/test_shortener_transform.py +++ b/rollbar/test/test_shortener_transform.py @@ -37,7 +37,9 @@ def setUp(self): 'frozenset': frozenset([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), 'array': array('l', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), 'deque': deque([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 15), - 'other': TestClassWithAVeryVeryVeryVeryVeryVeryVeryLongName() + 'other': TestClassWithAVeryVeryVeryVeryVeryVeryVeryLongName(), + 'list_max_level': [1, [2, [3, [4, ["good_5", ["bad_6", ["bad_7"]]]]]]], + 'dict_max_level': {1: 1, 2: {3: {4: {"level4": "good", "level5": {"toplevel": "ok", 6: {7: {}}}}}}} } def _assert_shortened(self, key, expected): @@ -46,13 +48,15 @@ def _assert_shortened(self, key, expected): if key == 'dict': self.assertEqual(expected, len(result)) + elif key in ('list_max_level', 'dict_max_level'): + self.assertEqual(expected, result[key]) else: # the repr output can vary between Python versions stripped_result_key = result[key].strip("'\"u") if key == 'other': self.assertIn(expected, stripped_result_key) - elif key != 'dict': + elif key not in ('dict', 'list_max_level', 'dict_max_level'): self.assertEqual(expected, stripped_result_key) # make sure nothing else was shortened @@ -82,6 +86,14 @@ def test_shorten_list(self): expected = '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...]' self._assert_shortened('list', expected) + def test_shorten_list_max_level(self): + expected = [1, [2, [3, [4, ['good_5']]]]] + self._assert_shortened('list_max_level', expected) + + def test_shorten_dict_max_level(self): + expected = {1: 1, 2: {3: {4: {'level4': 'good', 'level5': {'toplevel': 'ok'}}}}} + self._assert_shortened('dict_max_level', expected) + def test_shorten_tuple(self): expected = '(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...)' self._assert_shortened('tuple', expected) From debe4dc3a37faf9ed77111608a8c607d0c5d8f4b Mon Sep 17 00:00:00 2001 From: Pawel Date: Wed, 17 Apr 2024 00:30:10 -0700 Subject: [PATCH 8/9] fix bug in the previous test --- rollbar/test/test_shortener_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollbar/test/test_shortener_transform.py b/rollbar/test/test_shortener_transform.py index 3bcc458c..b1c3f796 100644 --- a/rollbar/test/test_shortener_transform.py +++ b/rollbar/test/test_shortener_transform.py @@ -47,7 +47,7 @@ def _assert_shortened(self, key, expected): result = transforms.transform(self.data, shortener) if key == 'dict': - self.assertEqual(expected, len(result)) + self.assertEqual(expected, len(result[key])) elif key in ('list_max_level', 'dict_max_level'): self.assertEqual(expected, result[key]) else: From e46e8e1983f9ee1c308e19f26acddfec17b6fda1 Mon Sep 17 00:00:00 2001 From: Pawel Date: Wed, 17 Apr 2024 00:42:53 -0700 Subject: [PATCH 9/9] added test for multi level shortening --- rollbar/test/test_shortener_transform.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/rollbar/test/test_shortener_transform.py b/rollbar/test/test_shortener_transform.py index b1c3f796..3dfe47d6 100644 --- a/rollbar/test/test_shortener_transform.py +++ b/rollbar/test/test_shortener_transform.py @@ -39,7 +39,8 @@ def setUp(self): 'deque': deque([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 15), 'other': TestClassWithAVeryVeryVeryVeryVeryVeryVeryLongName(), 'list_max_level': [1, [2, [3, [4, ["good_5", ["bad_6", ["bad_7"]]]]]]], - 'dict_max_level': {1: 1, 2: {3: {4: {"level4": "good", "level5": {"toplevel": "ok", 6: {7: {}}}}}}} + 'dict_max_level': {1: 1, 2: {3: {4: {"level4": "good", "level5": {"toplevel": "ok", 6: {7: {}}}}}}}, + 'list_multi_level': [1, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]] } def _assert_shortened(self, key, expected): @@ -48,7 +49,7 @@ def _assert_shortened(self, key, expected): if key == 'dict': self.assertEqual(expected, len(result[key])) - elif key in ('list_max_level', 'dict_max_level'): + elif key in ('list_max_level', 'dict_max_level', 'list_multi_level'): self.assertEqual(expected, result[key]) else: # the repr output can vary between Python versions @@ -56,7 +57,7 @@ def _assert_shortened(self, key, expected): if key == 'other': self.assertIn(expected, stripped_result_key) - elif key not in ('dict', 'list_max_level', 'dict_max_level'): + elif key not in ('dict', 'list_max_level', 'dict_max_level', 'list_multi_level'): self.assertEqual(expected, stripped_result_key) # make sure nothing else was shortened @@ -90,6 +91,10 @@ def test_shorten_list_max_level(self): expected = [1, [2, [3, [4, ['good_5']]]]] self._assert_shortened('list_max_level', expected) + def test_shorten_list_multi_level(self): + expected = [1, '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...]'] + self._assert_shortened('list_multi_level', expected) + def test_shorten_dict_max_level(self): expected = {1: 1, 2: {3: {4: {'level4': 'good', 'level5': {'toplevel': 'ok'}}}}} self._assert_shortened('dict_max_level', expected)