From 752a5e603431d35a4bb1e9d4c62e5e023d0fb3a2 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 13:36:40 +0200
Subject: [PATCH 01/17] implement custom (de)serialization for json and msgpack

---
 src/radical/utils/__init__.py         |  4 ++
 src/radical/utils/json_io.py          | 32 ++++++++++++++-
 src/radical/utils/zmq/utils.py        |  2 +
 tests/unittests/test_serialization.py | 56 +++++++++++++++++++++++++++
 4 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 tests/unittests/test_serialization.py

diff --git a/src/radical/utils/__init__.py b/src/radical/utils/__init__.py
index 8b7e57454..6c6d1bbe8 100644
--- a/src/radical/utils/__init__.py
+++ b/src/radical/utils/__init__.py
@@ -70,10 +70,14 @@
 
 from .json_io        import read_json, read_json_str, write_json
 from .json_io        import parse_json, parse_json_str, metric_expand
+from .json_io        import register_json_class, dumps_json
 from .which          import which
 from .tracer         import trace, untrace
 from .get_version    import get_version
 
+from .serialize      import to_json, from_json, to_msgpack, from_msgpack
+from .serialize      import register_serialization
+
 
 # import various utility methods
 from .ids            import *
diff --git a/src/radical/utils/json_io.py b/src/radical/utils/json_io.py
index 5eeac44b7..841035eb9 100644
--- a/src/radical/utils/json_io.py
+++ b/src/radical/utils/json_io.py
@@ -11,6 +11,23 @@
 from .misc import as_string, ru_open
 
 
+_json_classes = dict()
+
+def register_json_class(cls, check, convert):
+
+    global _json_classes
+    _json_classes[cls.__name__] = [check, convert]
+
+
+class _json_encoder(json.JSONEncoder):
+
+        def default(self, o):
+            for cls, (check, convert) in _json_classes.items():
+                if check(o):
+                    return convert(o)
+            return super().default(o)
+
+
 # ------------------------------------------------------------------------------
 #
 def read_json(fname, filter_comments=True):
@@ -61,11 +78,24 @@ def write_json(data, fname):
         fname = tmp
 
     with ru_open(fname, 'w') as f:
-        json.dump(data, f, sort_keys=True, indent=4, ensure_ascii=False)
+        json.dump(data, f, sort_keys=True, indent=4, ensure_ascii=False,
+                  cls=_json_encoder)
         f.write('\n')
         f.flush()
 
 
+# ------------------------------------------------------------------------------
+#
+def dumps_json(data):
+    '''
+    thin wrapper around python's json write, for consistency of interface
+
+    '''
+
+    return json.dumps(data, sort_keys=True, indent=4, ensure_ascii=False,
+                      cls=_json_encoder)
+
+
 # ------------------------------------------------------------------------------
 #
 def parse_json(json_str, filter_comments=True):
diff --git a/src/radical/utils/zmq/utils.py b/src/radical/utils/zmq/utils.py
index 4c39445be..2891cc6bd 100644
--- a/src/radical/utils/zmq/utils.py
+++ b/src/radical/utils/zmq/utils.py
@@ -3,6 +3,8 @@
 import zmq
 import errno
 
+import msgpack
+
 from ..url  import Url
 from ..misc import as_list
 from ..misc import ru_open
diff --git a/tests/unittests/test_serialization.py b/tests/unittests/test_serialization.py
new file mode 100644
index 000000000..fd491da7a
--- /dev/null
+++ b/tests/unittests/test_serialization.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+__author__    = "Radical.Utils Development Team (Andre Merzky)"
+__copyright__ = "Copyright 2024, RADICAL@Rutgers"
+__license__   = "MIT"
+
+import os
+
+import radical.utils as ru
+
+
+# ------------------------------------------------------------------------------
+#
+def test_serialization():
+
+    class Complex(object):
+
+        def __init__(self, real, imag):
+            self.real = real
+            self.imag = imag
+
+        def __eq__(self, other):
+            return self.real == other.real and self.imag == other.imag
+
+        def serialize(self):
+            return {'real': self.real, 'imag': self.imag}
+
+        @classmethod
+        def deserialize(cls, data):
+            return cls(data['real'], data['imag'])
+
+
+    ru.register_serialization(Complex, encode=Complex.serialize,
+                                       decode=Complex.deserialize)
+
+    data     = {'foo': {'complex_number': Complex(1, 2)}}
+    json_str = ru.to_json(data)
+    new_data = ru.from_json(json_str)
+
+    assert data == new_data
+
+    msgpack_str = ru.to_msgpack(data)
+    new_data    = ru.from_msgpack(msgpack_str)
+
+    assert data == new_data
+
+
+# ------------------------------------------------------------------------------
+#
+if __name__ == '__main__':
+
+    test_serialization()
+
+
+# ------------------------------------------------------------------------------
+

From 9e38697f976ad83e68db0adcbbd931c1eb2ea184 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 14:08:15 +0200
Subject: [PATCH 02/17] add missing file, use new methods

---
 src/radical/utils/__init__.py    |   3 +-
 src/radical/utils/json_io.py     |  76 ++--------------
 src/radical/utils/serialize.py   | 147 +++++++++++++++++++++++++++++++
 src/radical/utils/zmq/client.py  |  16 ++--
 src/radical/utils/zmq/message.py |   9 +-
 src/radical/utils/zmq/pipe.py    |   9 +-
 src/radical/utils/zmq/pubsub.py  |  32 +++----
 src/radical/utils/zmq/queue.py   |  46 +++++-----
 src/radical/utils/zmq/server.py  |  24 ++---
 src/radical/utils/zmq/utils.py   |   2 -
 tests/unittests/test_json.py     |  35 +-------
 11 files changed, 223 insertions(+), 176 deletions(-)
 create mode 100644 src/radical/utils/serialize.py

diff --git a/src/radical/utils/__init__.py b/src/radical/utils/__init__.py
index 6c6d1bbe8..f2f4621e4 100644
--- a/src/radical/utils/__init__.py
+++ b/src/radical/utils/__init__.py
@@ -69,8 +69,7 @@
 from .profile        import PROF_KEY_MAX
 
 from .json_io        import read_json, read_json_str, write_json
-from .json_io        import parse_json, parse_json_str, metric_expand
-from .json_io        import register_json_class, dumps_json
+from .json_io        import parse_json, parse_json_str, dumps_json
 from .which          import which
 from .tracer         import trace, untrace
 from .get_version    import get_version
diff --git a/src/radical/utils/json_io.py b/src/radical/utils/json_io.py
index 841035eb9..c2f768f73 100644
--- a/src/radical/utils/json_io.py
+++ b/src/radical/utils/json_io.py
@@ -8,24 +8,8 @@
 import re
 import json
 
-from .misc import as_string, ru_open
-
-
-_json_classes = dict()
-
-def register_json_class(cls, check, convert):
-
-    global _json_classes
-    _json_classes[cls.__name__] = [check, convert]
-
-
-class _json_encoder(json.JSONEncoder):
-
-        def default(self, o):
-            for cls, (check, convert) in _json_classes.items():
-                if check(o):
-                    return convert(o)
-            return super().default(o)
+from .serialize import to_json, from_json
+from .misc      import as_string, ru_open
 
 
 # ------------------------------------------------------------------------------
@@ -78,8 +62,7 @@ def write_json(data, fname):
         fname = tmp
 
     with ru_open(fname, 'w') as f:
-        json.dump(data, f, sort_keys=True, indent=4, ensure_ascii=False,
-                  cls=_json_encoder)
+        f.write(to_json(data))
         f.write('\n')
         f.flush()
 
@@ -92,8 +75,7 @@ def dumps_json(data):
 
     '''
 
-    return json.dumps(data, sort_keys=True, indent=4, ensure_ascii=False,
-                      cls=_json_encoder)
+    return to_json(data)
 
 
 # ------------------------------------------------------------------------------
@@ -116,7 +98,7 @@ def parse_json(json_str, filter_comments=True):
             content += re.sub(r'^\s*#.*$', '', line)
             content += '\n'
 
-        return json.loads(content)
+        return from_json(content)
 
 
 # ------------------------------------------------------------------------------
@@ -129,53 +111,5 @@ def parse_json_str(json_str):
     return as_string(parse_json(json_str))
 
 
-# ------------------------------------------------------------------------------
-#
-def metric_expand(data):
-    '''
-    iterate through the given dictionary, and when encountering a key string of
-    the form `ru.XYZ` or `rp.ABC`, expand them to their actually defined values.
-    This the following dict::
-
-        {
-            "ru.EVENT" : "foo"
-        }
-
-    becomes::
-
-        {
-            2 : "foo"
-        }
-    '''
-
-    try   : import radical.pilot as rp                              # noqa: F401
-    except: pass
-    try   : import radical.saga  as rs                              # noqa: F401
-    except: pass
-    try   : import radical.utils as ru                              # noqa: F401
-    except: pass
-
-    if isinstance(data, str):
-
-        if data.count('.') == 1:
-            elems = data.split('.')
-            if len(elems[0]) == 2 and elems[0][0] == 'r':
-                try:
-                    data = eval(data)
-                finally:
-
-                    pass
-        return data
-
-    elif isinstance(data, list):
-        return [metric_expand(elem) for elem in data]
-
-    elif isinstance(data, dict):
-        return {metric_expand(k) : metric_expand(v) for k,v in data.items()}
-
-    else:
-        return data
-
-
 # ------------------------------------------------------------------------------
 
diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
new file mode 100644
index 000000000..2edb6be37
--- /dev/null
+++ b/src/radical/utils/serialize.py
@@ -0,0 +1,147 @@
+
+import json
+import msgpack
+
+
+class _CType:
+
+    def __init__(self, ctype, encode, decode):
+
+        self.ctype : type     = ctype
+        self.encode: callable = encode
+        self.decode: callable = decode
+
+_ctypes = dict()
+
+
+# ------------------------------------------------------------------------------
+#
+def register_serialization(cls, encode, decode):
+    '''
+    register a class for json and msgpack serialization / deserialization.
+
+    Args:
+        cls    (type):     class type to register
+        encode (callable): converts class instance into encodable data structure
+        decode (callable): recreates the class instance from that data structure
+    '''
+
+    global _ctypes
+    _ctypes[cls.__name__] = _CType(cls, encode, decode)
+
+
+# ------------------------------------------------------------------------------
+#
+class _json_encoder(json.JSONEncoder):
+    '''
+    internal methods to encode registered classes to json
+    '''
+    def default(self, obj):
+        for cname,methods in _ctypes.items():
+            if isinstance(obj, methods.ctype):
+                return {'__%s__' % cname: True,
+                        'as_str'        : methods.encode(obj)}
+        return super().default(obj)
+
+# ------------------------------------------------------------------------------
+#
+def _json_decoder(obj):
+    '''
+    internal methods to decode registered classes from json
+    '''
+    for cname, methods in _ctypes.items():
+        if '__%s__' % cname in obj:
+            print('found %s' % cname)
+            return methods.decode(obj['as_str'])
+    return obj
+
+
+# ------------------------------------------------------------------------------
+#
+def _msgpack_encoder(obj):
+    '''
+    internal methods to encode registered classes to msgpack
+    '''
+    for cname,methods in _ctypes.items():
+        if isinstance(obj, methods.ctype):
+            return {'__%s__' % cname: True, 'as_str': methods.encode(obj)}
+    return obj
+
+
+# ------------------------------------------------------------------------------
+#
+def _msgpack_decoder(obj):
+    '''
+    internal methods to decode registered classes from msgpack
+    '''
+    for cname,methods in _ctypes.items():
+        if '__%s__' % cname in obj:
+            return methods.decode(obj['as_str'])
+    return obj
+
+
+# ------------------------------------------------------------------------------
+#
+def to_json(data):
+    '''
+    convert data to json, using registered classes for serialization
+
+    Args:
+        data (object): data to be serialized
+
+    Returns:
+        str: json serialized data
+    '''
+    return json.dumps(data, sort_keys=True, indent=4, ensure_ascii=False,
+                            cls=_json_encoder)
+
+
+# ------------------------------------------------------------------------------
+#
+def from_json(data):
+    '''
+    convert json data to python data structures, using registered classes for
+    deserialization
+
+    Args:
+        data (str): json data to be deserialized
+
+    Returns:
+        object: deserialized data
+    '''
+    return json.loads(data, object_hook=_json_decoder)
+
+
+# ------------------------------------------------------------------------------
+#
+def to_msgpack(data):
+    '''
+    convert data to msgpack, using registered classes for serialization
+
+    Args:
+        data (object): data to be serialized
+
+    Returns:
+        bytes: msgpack serialized data
+    '''
+    return msgpack.packb(data, default=_msgpack_encoder, use_bin_type=True)
+
+
+# ------------------------------------------------------------------------------
+#
+def from_msgpack(data):
+    '''
+    convert msgpack data to python data structures, using registered classes for
+    deserialization
+
+    Args:
+        data (bytes): msgpack data to be deserialized
+
+    Returns:
+        object: deserialized data
+    '''
+    return msgpack.unpackb(data, object_hook=_msgpack_decoder, raw=False)
+
+
+# ------------------------------------------------------------------------------
+
diff --git a/src/radical/utils/zmq/client.py b/src/radical/utils/zmq/client.py
index e23d17d23..4ec40aab0 100644
--- a/src/radical/utils/zmq/client.py
+++ b/src/radical/utils/zmq/client.py
@@ -1,14 +1,14 @@
 
 import zmq
-import msgpack
 
 from typing import Any
 
 import threading as mt
 
-from ..json_io import read_json
-from ..misc    import as_string
-from .utils    import no_intr, sock_connect
+from ..json_io   import read_json
+from ..misc      import as_string
+from ..serialize import to_msgpack, from_msgpack
+from .utils      import no_intr, sock_connect
 
 
 # ------------------------------------------------------------------------------
@@ -61,14 +61,14 @@ def url(self) -> str:
     #
     def request(self, cmd: str, *args: Any, **kwargs: Any) -> Any:
 
-        req = msgpack.packb({'cmd'   : cmd,
-                             'args'  : args,
-                             'kwargs': kwargs})
+        req = to_msgpack({'cmd'   : cmd,
+                          'args'  : args,
+                          'kwargs': kwargs})
 
         no_intr(self._sock.send, req)
 
         msg = no_intr(self._sock.recv)
-        res = as_string(msgpack.unpackb(msg))
+        res = as_string(from_msgpack(msg))
 
         # FIXME: assert proper res structure
 
diff --git a/src/radical/utils/zmq/message.py b/src/radical/utils/zmq/message.py
index b903f45ba..4ea3230ec 100644
--- a/src/radical/utils/zmq/message.py
+++ b/src/radical/utils/zmq/message.py
@@ -1,15 +1,16 @@
 
 from typing import Dict, Any
 
-import msgpack
-
 from ..typeddict import TypedDict
+from ..serialize import to_msgpack, from_msgpack, register_serialization
 
 
 # ------------------------------------------------------------------------------
 #
 class Message(TypedDict):
 
+    # FIXME: register serialization methods for all message types
+
     _schema = {
         '_msg_type': str,
     }
@@ -48,11 +49,11 @@ def deserialize(data: Dict[str, Any]):
 
 
     def packb(self):
-        return msgpack.packb(self)
+        return to_msgpack(self)
 
     @staticmethod
     def unpackb(bdata):
-        return Message.deserialize(msgpack.unpackb(bdata))
+        return Message.deserialize(from_msgpack(bdata))
 
 
 # ------------------------------------------------------------------------------
diff --git a/src/radical/utils/zmq/pipe.py b/src/radical/utils/zmq/pipe.py
index 09f71fa95..ec747fe33 100644
--- a/src/radical/utils/zmq/pipe.py
+++ b/src/radical/utils/zmq/pipe.py
@@ -1,6 +1,7 @@
 
 import zmq
-import msgpack
+
+from ..serialize import to_msgpack, from_msgpack
 
 MODE_PUSH = 'push'
 MODE_PULL = 'pull'
@@ -121,7 +122,7 @@ def put(self, msg):
         '''
 
         assert self._mode == MODE_PUSH
-        self._sock.send(msgpack.packb(msg))
+        self._sock.send(to_msgpack(msg))
 
 
     # --------------------------------------------------------------------------
@@ -132,7 +133,7 @@ def get(self):
         '''
 
         assert self._mode == MODE_PULL
-        return msgpack.unpackb(self._sock.recv())
+        return from_msgpack(self._sock.recv())
 
 
     # --------------------------------------------------------------------------
@@ -150,7 +151,7 @@ def get_nowait(self, timeout: float = 0):
         socks = dict(self._poller.poll(timeout=int(timeout * 1000)))
 
         if self._sock in socks:
-            return msgpack.unpackb(self._sock.recv())
+            return from_msgpack(self._sock.recv())
 
 
 # ------------------------------------------------------------------------------
diff --git a/src/radical/utils/zmq/pubsub.py b/src/radical/utils/zmq/pubsub.py
index 66f27475b..a4e350bbf 100644
--- a/src/radical/utils/zmq/pubsub.py
+++ b/src/radical/utils/zmq/pubsub.py
@@ -2,23 +2,23 @@
 
 import zmq
 import time
-import msgpack
 
 import threading as mt
 
-from typing    import Optional
+from typing      import Optional
 
-from ..atfork  import atfork
-from ..config  import Config
-from ..ids     import generate_id, ID_CUSTOM
-from ..url     import Url
-from ..misc    import as_string, as_bytes, as_list, noop
-from ..host    import get_hostip
-from ..logger  import Logger
-from ..profile import Profiler
+from ..atfork    import atfork
+from ..config    import Config
+from ..ids       import generate_id, ID_CUSTOM
+from ..url       import Url
+from ..misc      import as_string, as_bytes, as_list, noop
+from ..host      import get_hostip
+from ..logger    import Logger
+from ..profile   import Profiler
+from ..serialize import to_msgpack, from_msgpack
 
-from .bridge   import Bridge
-from .utils    import no_intr
+from .bridge     import Bridge
+from .utils      import no_intr
 
 
 # ------------------------------------------------------------------------------
@@ -246,7 +246,7 @@ def put(self, topic, msg):
       # log_bulk(self._log, '-> %s' % topic, [msg])
 
         btopic = as_bytes(topic.replace(' ', '_'))
-        bmsg   = msgpack.packb(msg)
+        bmsg   = to_msgpack(msg)
         data   = btopic + b' ' + bmsg
 
         self._socket.send(data)
@@ -273,7 +273,7 @@ def _get_nowait(socket, lock, timeout, log, prof):
 
             data        = no_intr(socket.recv, flags=zmq.NOBLOCK)
             topic, bmsg = data.split(b' ', 1)
-            msg         = msgpack.unpackb(bmsg)
+            msg         = from_msgpack(bmsg)
 
           # log.debug(' <- %s: %s', topic, msg)
 
@@ -497,7 +497,7 @@ def get(self):
             data = no_intr(self._sock.recv)
 
         topic, bmsg = data.split(b' ', 1)
-        msg = msgpack.unpackb(bmsg)
+        msg = from_msgpack(bmsg)
 
       # log_bulk(self._log, '<- %s' % topic, [msg])
 
@@ -519,7 +519,7 @@ def get_nowait(self, timeout=None):
                 data = no_intr(self._sock.recv, flags=zmq.NOBLOCK)
 
             topic, bmsg = data.split(b' ', 1)
-            msg = msgpack.unpackb(bmsg)
+            msg = from_msgpack(bmsg)
 
           # log_bulk(self._log, '<- %s' % topic, [msg])
 
diff --git a/src/radical/utils/zmq/queue.py b/src/radical/utils/zmq/queue.py
index 9914fb19e..bb90ee052 100644
--- a/src/radical/utils/zmq/queue.py
+++ b/src/radical/utils/zmq/queue.py
@@ -3,24 +3,24 @@
 import sys
 import zmq
 import time
-import msgpack
 
 import threading as mt
 
-from typing import Optional
+from typing      import Optional
 
-from ..atfork  import atfork
-from ..config  import Config
-from ..ids     import generate_id, ID_CUSTOM
-from ..url     import Url
-from ..misc    import as_string, as_bytes, as_list, noop
-from ..host    import get_hostip
-from ..logger  import Logger
-from ..profile import Profiler
-from ..debug   import print_exception_trace
+from ..atfork    import atfork
+from ..config    import Config
+from ..ids       import generate_id, ID_CUSTOM
+from ..url       import Url
+from ..misc      import as_string, as_bytes, as_list, noop
+from ..host      import get_hostip
+from ..logger    import Logger
+from ..profile   import Profiler
+from ..debug     import print_exception_trace
+from ..serialize import to_msgpack, from_msgpack
 
-from .bridge   import Bridge
-from .utils    import no_intr
+from .bridge     import Bridge
+from .utils      import no_intr
 
 # NOTE: the log bulk method is frequently called and slow
 # from .utils    import log_bulk
@@ -238,8 +238,8 @@ def _bridge_work(self):
                     if len(data) != 2:
                         raise RuntimeError('%d frames unsupported' % len(data))
 
-                    qname = as_string(msgpack.unpackb(data[0]))
-                    msgs  = msgpack.unpackb(data[1])
+                    qname = as_string(from_msgpack(data[0]))
+                    msgs  = from_msgpack(data[1])
                   # prof_bulk(self._prof, 'poll_put_recv', msgs)
                   # log_bulk(self._log, '<> %s' % qname, msgs)
                   # self._log.debug('put %s: %s ! ', qname, len(msgs))
@@ -278,7 +278,7 @@ def _bridge_work(self):
 
                   # log_bulk(self._log, '>< %s' % qname, msgs)
 
-                    data   = [msgpack.packb(qname), msgpack.packb(msgs)]
+                    data   = [to_msgpack(qname), to_msgpack(msgs)]
                     active = True
 
                   # self._log.debug('==== get %s: %s', qname, list(buf.keys()))
@@ -380,7 +380,7 @@ def put(self, msgs, qname=None):
             qname = 'default'
 
       # log_bulk(self._log, '-> %s[%s]' % (self._channel, qname), msgs)
-        data = [msgpack.packb(qname), msgpack.packb(msgs)]
+        data = [to_msgpack(qname), to_msgpack(msgs)]
 
         with self._lock:
             no_intr(self._q.send_multipart, data)
@@ -427,8 +427,8 @@ def _get_nowait(url, qname=None, timeout=None, uid=None):  # timeout in ms
                 data = list(no_intr(info['socket'].recv_multipart))
                 info['requested'] = False
 
-                qname = as_string(msgpack.unpackb(data[0]))
-                msgs  = as_string(msgpack.unpackb(data[1]))
+                qname = as_string(from_msgpack(data[0]))
+                msgs  = as_string(from_msgpack(data[1]))
               # log_bulk(logger, '<-1 %s [%s]' % (uid, qname), msgs)
                 return msgs
 
@@ -694,8 +694,8 @@ def get(self, qname=None):
             data = list(no_intr(self._q.recv_multipart))
             self._requested = False
 
-        qname = msgpack.unpackb(data[0])
-        msgs  = msgpack.unpackb(data[1])
+        qname = from_msgpack(data[0])
+        msgs  = from_msgpack(data[1])
 
       # log_bulk(self._log, '<-2 %s [%s]' % (self._channel, qname), msgs)
 
@@ -729,8 +729,8 @@ def get_nowait(self, qname=None, timeout=None):  # timeout in ms
                 data = list(no_intr(self._q.recv_multipart))
                 self._requested = False
 
-            qname = msgpack.unpackb(data[0])
-            msgs  = msgpack.unpackb(data[1])
+            qname = from_msgpack(data[0])
+            msgs  = from_msgpack(data[1])
           # log_bulk(self._log, '<-3 %s [%s]' % (self._channel, qname), msgs)
 
             return as_string(msgs)
diff --git a/src/radical/utils/zmq/server.py b/src/radical/utils/zmq/server.py
index 69eac8864..74492104b 100644
--- a/src/radical/utils/zmq/server.py
+++ b/src/radical/utils/zmq/server.py
@@ -1,20 +1,20 @@
 
 import zmq
-import msgpack
 
 import threading as mt
 
-from typing import Optional, Union, Iterator, Any, Dict
+from typing      import Optional, Union, Iterator, Any, Dict
 
-from ..ids     import generate_id
-from ..url     import Url
-from ..misc    import as_string
-from ..host    import get_hostip
-from ..logger  import Logger
-from ..profile import Profiler
-from ..debug   import get_exception_trace
+from ..ids       import generate_id
+from ..url       import Url
+from ..misc      import as_string
+from ..host      import get_hostip
+from ..logger    import Logger
+from ..profile   import Profiler
+from ..debug     import get_exception_trace
+from ..serialize import to_msgpack, from_msgpack
 
-from .utils    import no_intr
+from .utils      import no_intr
 
 
 # --------------------------------------------------------------------------
@@ -284,7 +284,7 @@ def _work(self) -> None:
 
             try:
                 data = no_intr(self._sock.recv)
-                req  = as_string(msgpack.unpackb(data))
+                req  = as_string(from_msgpack(data))
                 self._log.debug('req: %s', str(req)[:128])
 
                 if not isinstance(req, dict):
@@ -312,7 +312,7 @@ def _work(self) -> None:
             finally:
                 if not rep:
                     rep = self._error('server error')
-                no_intr(self._sock.send, msgpack.packb(rep))
+                no_intr(self._sock.send, to_msgpack(rep))
                 self._log.debug('rep: %s', str(rep)[:128])
 
         self._sock.close()
diff --git a/src/radical/utils/zmq/utils.py b/src/radical/utils/zmq/utils.py
index 2891cc6bd..4c39445be 100644
--- a/src/radical/utils/zmq/utils.py
+++ b/src/radical/utils/zmq/utils.py
@@ -3,8 +3,6 @@
 import zmq
 import errno
 
-import msgpack
-
 from ..url  import Url
 from ..misc import as_list
 from ..misc import ru_open
diff --git a/tests/unittests/test_json.py b/tests/unittests/test_json.py
index c0e111f65..a2c0e5645 100644
--- a/tests/unittests/test_json.py
+++ b/tests/unittests/test_json.py
@@ -6,44 +6,11 @@
 import radical.utils as ru
 
 
-# ------------------------------------------------------------------------------
-#
-def test_metric_expand():
-
-    d_in  = {'total'     : [{'ru.EVENT': 'bootstrap_0_start'},
-                            {'ru.EVENT': 'bootstrap_0_stop' }],
-             'boot'      : [{'ru.EVENT': 'bootstrap_0_start'},
-                            {'ru.EVENT': 'sync_rel'         }],
-             'setup_1'   : [{'ru.EVENT': 'sync_rel'         },
-                            {'ru.STATE': 'rp.PMGR_ACTIVE'   }],
-             'ignore'    : [{'ru.STATE': 'rp.PMGR_ACTIVE'   },
-                            {'ru.EVENT': 'cmd'              ,
-                             'ru.MSG  ': 'cancel_pilot'     }],
-             'term'      : [{'ru.EVENT': 'cmd'              ,
-                             'ru.MSG  ': 'cancel_pilot'     },
-                            {'ru.EVENT': 'bootstrap_0_stop' }]}
-
-    d_out = {'total'     : [{1         : 'bootstrap_0_start'},
-                            {1         : 'bootstrap_0_stop' }],
-             'boot'      : [{1         : 'bootstrap_0_start'},
-                            {1         : 'sync_rel'         }],
-             'setup_1'   : [{1         : 'sync_rel'         },
-                            {5         : 'PMGR_ACTIVE'      }],
-             'ignore'    : [{5         : 'PMGR_ACTIVE'      },
-                            {1         : 'cmd'              ,
-                             6         : 'cancel_pilot'     }],
-             'term'      : [{1         : 'cmd'              ,
-                             6         : 'cancel_pilot'     },
-                            {1         : 'bootstrap_0_stop' }]}
-
-    assert ru.metric_expand(d_in) == d_out
-
-
 # ------------------------------------------------------------------------------
 # run tests if called directly
 if __name__ == "__main__":
 
-    test_metric_expand()
+    pass
 
 
 # ------------------------------------------------------------------------------

From 1e686e60c109e51ac8bd5bff81b262edee172eb1 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 14:12:50 +0200
Subject: [PATCH 03/17] method rename

---
 src/radical/utils/__init__.py         | 2 +-
 src/radical/utils/serialize.py        | 2 +-
 src/radical/utils/zmq/message.py      | 2 +-
 tests/unittests/test_serialization.py | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/radical/utils/__init__.py b/src/radical/utils/__init__.py
index f2f4621e4..be8e0352d 100644
--- a/src/radical/utils/__init__.py
+++ b/src/radical/utils/__init__.py
@@ -75,7 +75,7 @@
 from .get_version    import get_version
 
 from .serialize      import to_json, from_json, to_msgpack, from_msgpack
-from .serialize      import register_serialization
+from .serialize      import register_serializable
 
 
 # import various utility methods
diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
index 2edb6be37..a644eb13f 100644
--- a/src/radical/utils/serialize.py
+++ b/src/radical/utils/serialize.py
@@ -16,7 +16,7 @@ def __init__(self, ctype, encode, decode):
 
 # ------------------------------------------------------------------------------
 #
-def register_serialization(cls, encode, decode):
+def register_serializable(cls, encode, decode):
     '''
     register a class for json and msgpack serialization / deserialization.
 
diff --git a/src/radical/utils/zmq/message.py b/src/radical/utils/zmq/message.py
index 4ea3230ec..01661af15 100644
--- a/src/radical/utils/zmq/message.py
+++ b/src/radical/utils/zmq/message.py
@@ -2,7 +2,7 @@
 from typing import Dict, Any
 
 from ..typeddict import TypedDict
-from ..serialize import to_msgpack, from_msgpack, register_serialization
+from ..serialize import to_msgpack, from_msgpack, register_serializable
 
 
 # ------------------------------------------------------------------------------
diff --git a/tests/unittests/test_serialization.py b/tests/unittests/test_serialization.py
index fd491da7a..493b37794 100644
--- a/tests/unittests/test_serialization.py
+++ b/tests/unittests/test_serialization.py
@@ -30,8 +30,8 @@ def deserialize(cls, data):
             return cls(data['real'], data['imag'])
 
 
-    ru.register_serialization(Complex, encode=Complex.serialize,
-                                       decode=Complex.deserialize)
+    ru.register_serializable(Complex, encode=Complex.serialize,
+                                      decode=Complex.deserialize)
 
     data     = {'foo': {'complex_number': Complex(1, 2)}}
     json_str = ru.to_json(data)

From 571d00b29180ea550e7638b70de7e2ef6f45b285 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 16:21:20 +0200
Subject: [PATCH 04/17] make all typed dicts json (de)serializable

---
 src/radical/utils/serialize.py | 33 +++++++++++++++++++++++++++------
 src/radical/utils/typeddict.py |  5 ++++-
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
index a644eb13f..fc5d51e14 100644
--- a/src/radical/utils/serialize.py
+++ b/src/radical/utils/serialize.py
@@ -1,4 +1,5 @@
 
+import copy
 import json
 import msgpack
 
@@ -16,7 +17,7 @@ def __init__(self, ctype, encode, decode):
 
 # ------------------------------------------------------------------------------
 #
-def register_serializable(cls, encode, decode):
+def register_serializable(cls, encode=None, decode=None):
     '''
     register a class for json and msgpack serialization / deserialization.
 
@@ -26,21 +27,36 @@ def register_serializable(cls, encode, decode):
         decode (callable): recreates the class instance from that data structure
     '''
 
+    if encode is None: encode = cls
+    if decode is None: decode = cls
+
     global _ctypes
     _ctypes[cls.__name__] = _CType(cls, encode, decode)
 
 
 # ------------------------------------------------------------------------------
 #
+
 class _json_encoder(json.JSONEncoder):
     '''
     internal methods to encode registered classes to json
     '''
+
+    def encode(self, o, *args, **kw):
+
+        from .typeddict import TypedDict
+        if isinstance(o, TypedDict):
+          # print('TypedDict: %s' % type(o).__name__)
+            o = copy.deepcopy(o)
+            o['_xtype'] = type(o).__name__
+        return super().encode(o, *args, **kw)
+
     def default(self, obj):
+      # print('encode: %s' % obj)
         for cname,methods in _ctypes.items():
             if isinstance(obj, methods.ctype):
-                return {'__%s__' % cname: True,
-                        'as_str'        : methods.encode(obj)}
+                return {'_xtype': cname,
+                        'as_str': methods.encode(obj)}
         return super().default(obj)
 
 # ------------------------------------------------------------------------------
@@ -49,10 +65,15 @@ def _json_decoder(obj):
     '''
     internal methods to decode registered classes from json
     '''
+  # print('decode: %s' % obj)
     for cname, methods in _ctypes.items():
-        if '__%s__' % cname in obj:
-            print('found %s' % cname)
-            return methods.decode(obj['as_str'])
+      # print('check %s' % cname)
+        if '_xtype' in obj and obj['_xtype'] == cname:
+            del obj['_xtype']
+          # print('found %s' % cname)
+            if 'as_str' in obj:
+                return methods.decode(obj['as_str'])
+            return methods.decode(obj)
     return obj
 
 
diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 0cf9c284a..fea6d21f8 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -20,7 +20,8 @@
 import copy
 import sys
 
-from .misc import as_list, as_tuple, is_string
+from .serialize import register_serializable
+from .misc      import as_list, as_tuple, is_string
 
 
 # ------------------------------------------------------------------------------
@@ -138,6 +139,8 @@ def __init__(self, from_dict=None, **kwargs):
               `kwargs`).
         '''
 
+        register_serializable(self.__class__)
+
         self.update(copy.deepcopy(self._defaults))
         self.update(from_dict)
 

From 6477fe12ef0fe2774dfd1e04c2e29899a2ef1113 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 16:24:24 +0200
Subject: [PATCH 05/17] update test

---
 tests/unittests/test_serialization.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/unittests/test_serialization.py b/tests/unittests/test_serialization.py
index 493b37794..489b52466 100644
--- a/tests/unittests/test_serialization.py
+++ b/tests/unittests/test_serialization.py
@@ -44,6 +44,21 @@ def deserialize(cls, data):
 
     assert data == new_data
 
+    class A(ru.TypedDict):
+        _schema = {'s': str,
+                   'i': int}
+
+    class B(ru.TypedDict):
+        _schema = {'a': A}
+
+    old_data = B(a=A(i=42, s='buz'))
+    tmp      = ru.to_json(old_data)
+    new_data = ru.from_json(tmp )
+
+    assert old_data == new_data
+    assert type(old_data) == type(new_data)
+    assert type(old_data['a']) == type(new_data['a'])
+
 
 # ------------------------------------------------------------------------------
 #

From 579477441681692a9d88976060133e68149bf688 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 16:26:25 +0200
Subject: [PATCH 06/17] streamline

---
 tests/unittests/test_serialization.py | 30 +++++++++++++--------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tests/unittests/test_serialization.py b/tests/unittests/test_serialization.py
index 489b52466..58bb1e227 100644
--- a/tests/unittests/test_serialization.py
+++ b/tests/unittests/test_serialization.py
@@ -33,31 +33,30 @@ def deserialize(cls, data):
     ru.register_serializable(Complex, encode=Complex.serialize,
                                       decode=Complex.deserialize)
 
-    data     = {'foo': {'complex_number': Complex(1, 2)}}
-    json_str = ru.to_json(data)
-    new_data = ru.from_json(json_str)
+    old = {'foo': {'complex_number': Complex(1, 2)}}
+    new = ru.from_json(ru.to_json(old))
 
-    assert data == new_data
+    assert old == new
 
-    msgpack_str = ru.to_msgpack(data)
-    new_data    = ru.from_msgpack(msgpack_str)
+    new = ru.from_msgpack(ru.to_msgpack(old))
 
-    assert data == new_data
+    assert old == new
+
+
+def test_serialization_typed_dict():
 
     class A(ru.TypedDict):
-        _schema = {'s': str,
-                   'i': int}
+        _schema = {'s': str, 'i': int}
 
     class B(ru.TypedDict):
         _schema = {'a': A}
 
-    old_data = B(a=A(i=42, s='buz'))
-    tmp      = ru.to_json(old_data)
-    new_data = ru.from_json(tmp )
+    old = B(a=A(i=42, s='buz'))
+    new = ru.from_json(ru.to_json(old))
 
-    assert old_data == new_data
-    assert type(old_data) == type(new_data)
-    assert type(old_data['a']) == type(new_data['a'])
+    assert old == new
+    assert type(old) == type(new)
+    assert type(old['a']) == type(new['a'])
 
 
 # ------------------------------------------------------------------------------
@@ -65,6 +64,7 @@ class B(ru.TypedDict):
 if __name__ == '__main__':
 
     test_serialization()
+    test_serialization_typed_dict()
 
 
 # ------------------------------------------------------------------------------

From 75f5747acfe2a89bfa9df7327c37ae6832befc12 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 16:27:26 +0200
Subject: [PATCH 07/17] linting

---
 tests/unittests/test_serialization.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unittests/test_serialization.py b/tests/unittests/test_serialization.py
index 58bb1e227..77fbd9fd6 100644
--- a/tests/unittests/test_serialization.py
+++ b/tests/unittests/test_serialization.py
@@ -4,8 +4,6 @@
 __copyright__ = "Copyright 2024, RADICAL@Rutgers"
 __license__   = "MIT"
 
-import os
-
 import radical.utils as ru
 
 
@@ -43,6 +41,8 @@ def deserialize(cls, data):
     assert old == new
 
 
+# ------------------------------------------------------------------------------
+#
 def test_serialization_typed_dict():
 
     class A(ru.TypedDict):
@@ -51,7 +51,7 @@ class A(ru.TypedDict):
     class B(ru.TypedDict):
         _schema = {'a': A}
 
-    old = B(a=A(i=42, s='buz'))
+    old = B(a=A(s='buz', i=42))
     new = ru.from_json(ru.to_json(old))
 
     assert old == new

From 6dcd239f23193ff478a4e7b350557058c43231f8 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Fri, 5 Apr 2024 16:36:55 +0200
Subject: [PATCH 08/17] fix deep iteration

---
 src/radical/utils/serialize.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
index fc5d51e14..55f476ff6 100644
--- a/src/radical/utils/serialize.py
+++ b/src/radical/utils/serialize.py
@@ -36,19 +36,30 @@ def register_serializable(cls, encode=None, decode=None):
 
 # ------------------------------------------------------------------------------
 #
+def _prep_typed_dict(d):
+    from .typeddict import TypedDict
+    tmp = dict()
+    for k,v in d.items():
+        if isinstance(v, TypedDict):
+            tmp[k] = _prep_typed_dict(v)
+        else:
+            tmp[k] = v
+    tmp['_xtype'] = type(d).__name__
+    return tmp
 
+
+# ------------------------------------------------------------------------------
+#
 class _json_encoder(json.JSONEncoder):
     '''
     internal methods to encode registered classes to json
     '''
 
     def encode(self, o, *args, **kw):
-
         from .typeddict import TypedDict
         if isinstance(o, TypedDict):
-          # print('TypedDict: %s' % type(o).__name__)
-            o = copy.deepcopy(o)
-            o['_xtype'] = type(o).__name__
+            tmp = _prep_typed_dict(o)
+            return super().encode(tmp, *args, **kw)
         return super().encode(o, *args, **kw)
 
     def default(self, obj):

From cf31a3a79e1dc4dd848fb2f4f4c7e8e0327bb258 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 6 Apr 2024 00:52:25 +0200
Subject: [PATCH 09/17] simpler code

---
 src/radical/utils/serialize.py | 28 +++++++++++-----------------
 src/radical/utils/typeddict.py | 30 +++++++++++++++++++-----------
 2 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
index 55f476ff6..d87969ba9 100644
--- a/src/radical/utils/serialize.py
+++ b/src/radical/utils/serialize.py
@@ -3,7 +3,11 @@
 import json
 import msgpack
 
+from .typeddict import as_dict
 
+
+# ------------------------------------------------------------------------------
+#
 class _CType:
 
     def __init__(self, ctype, encode, decode):
@@ -37,15 +41,8 @@ def register_serializable(cls, encode=None, decode=None):
 # ------------------------------------------------------------------------------
 #
 def _prep_typed_dict(d):
-    from .typeddict import TypedDict
-    tmp = dict()
-    for k,v in d.items():
-        if isinstance(v, TypedDict):
-            tmp[k] = _prep_typed_dict(v)
-        else:
-            tmp[k] = v
-    tmp['_xtype'] = type(d).__name__
-    return tmp
+    from .typeddict import as_dict
+    return as_dict(d, _annotate=True)
 
 
 # ------------------------------------------------------------------------------
@@ -56,17 +53,14 @@ class _json_encoder(json.JSONEncoder):
     '''
 
     def encode(self, o, *args, **kw):
-        from .typeddict import TypedDict
-        if isinstance(o, TypedDict):
-            tmp = _prep_typed_dict(o)
-            return super().encode(tmp, *args, **kw)
-        return super().encode(o, *args, **kw)
+        tmp = as_dict(o, _annotate=True)
+        return super().encode(tmp, *args, **kw)
 
     def default(self, obj):
       # print('encode: %s' % obj)
         for cname,methods in _ctypes.items():
             if isinstance(obj, methods.ctype):
-                return {'_xtype': cname,
+                return {'_type': cname,
                         'as_str': methods.encode(obj)}
         return super().default(obj)
 
@@ -79,8 +73,8 @@ def _json_decoder(obj):
   # print('decode: %s' % obj)
     for cname, methods in _ctypes.items():
       # print('check %s' % cname)
-        if '_xtype' in obj and obj['_xtype'] == cname:
-            del obj['_xtype']
+        if '_type' in obj and obj['_type'] == cname:
+            del obj['_type']
           # print('found %s' % cname)
             if 'as_str' in obj:
                 return methods.decode(obj['as_str'])
diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index fea6d21f8..2173f140d 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -17,10 +17,10 @@
 #   - optional runtime type checking
 #
 
+import collections
 import copy
 import sys
 
-from .serialize import register_serializable
 from .misc      import as_list, as_tuple, is_string
 
 
@@ -139,6 +139,8 @@ def __init__(self, from_dict=None, **kwargs):
               `kwargs`).
         '''
 
+        from .serialize import register_serializable
+
         register_serializable(self.__class__)
 
         self.update(copy.deepcopy(self._defaults))
@@ -315,8 +317,8 @@ def __repr__(self):
 
     # --------------------------------------------------------------------------
     #
-    def as_dict(self):
-        return as_dict(self._data)
+    def as_dict(self, _annotate=False):
+        return as_dict(self._data, _annotate)
 
 
     # --------------------------------------------------------------------------
@@ -486,21 +488,27 @@ def _query(self, key, default=None, last_key=True):
 
 # ------------------------------------------------------------------------------
 #
-def _as_dict_value(v):
-    return v.as_dict() if isinstance(v, TypedDict) else as_dict(v)
+def _as_dict_value(v, _annotate=False):
+    if isinstance(v, TypedDict):
+        ret = copy.deepcopy(v)
+        if _annotate:
+            ret['_type'] = type(v).__name__
+        return ret
+    else:
+        return as_dict(v, _annotate)
 
 
-def as_dict(src):
+def as_dict(src, _annotate=False):
     '''
     Iterate given object, apply `as_dict()` to all typed
     values, and return the result (effectively a shallow copy).
     '''
     if isinstance(src, dict):
-        tgt = {k: _as_dict_value(v) for k, v in src.items()}
-    elif isinstance(src, list):
-        tgt = [_as_dict_value(x) for x in src]
-    elif isinstance(src, tuple):
-        tgt = tuple([_as_dict_value(x) for x in src])
+        tgt = {k: _as_dict_value(v, _annotate) for k, v in src.items()}
+        if _annotate:
+            tgt['_type'] = type(src).__name__
+    elif isinstance(src, collections.abc.Iterable):
+        tgt = [_as_dict_value(x, _annotate) for x in src]
     else:
         tgt = src
     return tgt

From b5af2d0a88a84b3ce86ecc6c30ce3ce2cf6a48b0 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 6 Apr 2024 08:20:42 +0200
Subject: [PATCH 10/17] typed  dict classes should have from_dict as default
 arg

---
 src/radical/utils/config.py    | 15 +++++++++++----
 src/radical/utils/typeddict.py | 22 ++++++++--------------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/radical/utils/config.py b/src/radical/utils/config.py
index f795f540c..37c11f12c 100644
--- a/src/radical/utils/config.py
+++ b/src/radical/utils/config.py
@@ -176,22 +176,23 @@ class Config(TypedDict):
 
     # --------------------------------------------------------------------------
     #
-    def __init__(self, module=None, category=None, name=None, cfg=None,
-                       from_dict=None, path=None, expand=True, env=None,
-                       _internal=False):
+    def __init__(self, from_dict=None,
+                       module=None, category=None, name=None,
+                       cfg=None,    path=None,     expand=True,
+                       env=None, _internal=False):
         """
         Load a config (json) file from the module's config tree, and overload
         any user specific config settings if found.
 
         Parameters
         ----------
+        from_dict: alias for cfg, to satisfy base class constructor
         module:    used to determine the module's config file location
                    - default: `radical.utils`
         category:  name of config to be loaded from module's config path
         name:      specify a specific configuration to be used
         path:      path to app config json to be used for initialization
         cfg:       application config dict to be used for initialization
-        from_dict: alias for cfg, to satisfy base class constructor
         expand:    enable / disable environment var expansion
                    - default: True
         env:       environment dictionary to be used for expansion
@@ -215,6 +216,12 @@ def __init__(self, module=None, category=None, name=None, cfg=None,
               configuration hierarchy.
         """
 
+        # if the `from_dict` is given but is a string, we interpret it as
+        # `module` parameter.
+        if from_dict and isinstance(from_dict, str):
+            module    = from_dict
+            from_dict = None
+
         if from_dict:
             # if we could only overload constructors by signature... :-/
             # As it is, we have to emulate that...
diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 2173f140d..5d606827f 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -488,27 +488,21 @@ def _query(self, key, default=None, last_key=True):
 
 # ------------------------------------------------------------------------------
 #
-def _as_dict_value(v, _annotate=False):
-    if isinstance(v, TypedDict):
-        ret = copy.deepcopy(v)
-        if _annotate:
-            ret['_type'] = type(v).__name__
-        return ret
-    else:
-        return as_dict(v, _annotate)
-
-
 def as_dict(src, _annotate=False):
     '''
     Iterate given object, apply `as_dict()` to all typed
     values, and return the result (effectively a shallow copy).
     '''
-    if isinstance(src, dict):
-        tgt = {k: _as_dict_value(v, _annotate) for k, v in src.items()}
+    if isinstance(src, TypedDict):
+        tgt = {k: as_dict(v, _annotate) for k, v in src.items()}
         if _annotate:
             tgt['_type'] = type(src).__name__
-    elif isinstance(src, collections.abc.Iterable):
-        tgt = [_as_dict_value(x, _annotate) for x in src]
+    elif isinstance(src, dict):
+        tgt = {k: as_dict(v, _annotate) for k, v in src.items()}
+    elif isinstance(src, list):
+        tgt = [as_dict(x, _annotate) for x in src]
+    elif isinstance(src, tuple):
+        tgt = tuple([as_dict(x, _annotate) for x in src])
     else:
         tgt = src
     return tgt

From 9095c338f97b5afc973bbd39954b8941a0b52022 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 6 Apr 2024 21:22:20 +0200
Subject: [PATCH 11/17] linting

---
 src/radical/utils/serialize.py        | 15 +++++++--------
 src/radical/utils/typeddict.py        |  1 -
 src/radical/utils/zmq/message.py      |  2 +-
 tests/unittests/test_json.py          |  2 +-
 tests/unittests/test_serialization.py |  4 ++--
 5 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
index d87969ba9..61c8072fe 100644
--- a/src/radical/utils/serialize.py
+++ b/src/radical/utils/serialize.py
@@ -1,5 +1,4 @@
 
-import copy
 import json
 import msgpack
 
@@ -16,6 +15,7 @@ def __init__(self, ctype, encode, decode):
         self.encode: callable = encode
         self.decode: callable = decode
 
+
 _ctypes = dict()
 
 
@@ -34,14 +34,12 @@ def register_serializable(cls, encode=None, decode=None):
     if encode is None: encode = cls
     if decode is None: decode = cls
 
-    global _ctypes
     _ctypes[cls.__name__] = _CType(cls, encode, decode)
 
 
 # ------------------------------------------------------------------------------
 #
 def _prep_typed_dict(d):
-    from .typeddict import as_dict
     return as_dict(d, _annotate=True)
 
 
@@ -56,13 +54,14 @@ def encode(self, o, *args, **kw):
         tmp = as_dict(o, _annotate=True)
         return super().encode(tmp, *args, **kw)
 
-    def default(self, obj):
-      # print('encode: %s' % obj)
+    def default(self, o):
+      # print('encode: %s' % o)
         for cname,methods in _ctypes.items():
-            if isinstance(obj, methods.ctype):
+            if isinstance(o, methods.ctype):
                 return {'_type': cname,
-                        'as_str': methods.encode(obj)}
-        return super().default(obj)
+                        'as_str': methods.encode(o)}
+        return super().default(o)
+
 
 # ------------------------------------------------------------------------------
 #
diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 5d606827f..87ceb6c7b 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -17,7 +17,6 @@
 #   - optional runtime type checking
 #
 
-import collections
 import copy
 import sys
 
diff --git a/src/radical/utils/zmq/message.py b/src/radical/utils/zmq/message.py
index 01661af15..1fa30bdf2 100644
--- a/src/radical/utils/zmq/message.py
+++ b/src/radical/utils/zmq/message.py
@@ -2,7 +2,7 @@
 from typing import Dict, Any
 
 from ..typeddict import TypedDict
-from ..serialize import to_msgpack, from_msgpack, register_serializable
+from ..serialize import to_msgpack, from_msgpack
 
 
 # ------------------------------------------------------------------------------
diff --git a/tests/unittests/test_json.py b/tests/unittests/test_json.py
index a2c0e5645..138cd3a4e 100644
--- a/tests/unittests/test_json.py
+++ b/tests/unittests/test_json.py
@@ -3,7 +3,7 @@
 # noqa: E201
 
 
-import radical.utils as ru
+# import radical.utils as ru
 
 
 # ------------------------------------------------------------------------------
diff --git a/tests/unittests/test_serialization.py b/tests/unittests/test_serialization.py
index 77fbd9fd6..b685171fd 100644
--- a/tests/unittests/test_serialization.py
+++ b/tests/unittests/test_serialization.py
@@ -55,8 +55,8 @@ class B(ru.TypedDict):
     new = ru.from_json(ru.to_json(old))
 
     assert old == new
-    assert type(old) == type(new)
-    assert type(old['a']) == type(new['a'])
+    assert isinstance(old, B)      and isinstance(new, B)
+    assert isinstance(old['a'], A) and isinstance(new['a'], A)
 
 
 # ------------------------------------------------------------------------------

From 91617e53fb85fcf8de64e9c912f6ce7a6d67656f Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sun, 7 Apr 2024 21:36:08 +0200
Subject: [PATCH 12/17] ignore underunderscored attributes

---
 src/radical/utils/typeddict.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 87ceb6c7b..49e4ff119 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -292,15 +292,15 @@ def __getattr__(self, k):
 
     def __setattr__(self, k, v):
 
-        # if k.startswith('_'):
-        #     return object.__setattr__(self, k, v)
+        if k.startswith('__'):
+            return object.__setattr__(self, k, v)
 
         self._data[k] = self._verify_setter(k, v)
 
     def __delattr__(self, k):
 
-        # if k.startswith('_'):
-        #     return object.__delattr__(self, k)
+        if k.startswith('__'):
+            return object.__delattr__(self, k)
 
         del self._data[k]
 

From 997c9705e74f445f57c2c00bee407c7c5f41f298 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sun, 7 Apr 2024 21:53:00 +0200
Subject: [PATCH 13/17] ignore underscored attributes

---
 src/radical/utils/typeddict.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 49e4ff119..4bf38b220 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -280,7 +280,7 @@ def __getattr__(self, k):
                 self.__dict__['_data'] = dict()
             return self.__dict__['_data']
 
-        if k.startswith('__'):
+        if k.startswith('_'):
             return object.__getattribute__(self, k)
 
         data   = self._data
@@ -292,14 +292,14 @@ def __getattr__(self, k):
 
     def __setattr__(self, k, v):
 
-        if k.startswith('__'):
+        if k.startswith('_'):
             return object.__setattr__(self, k, v)
 
         self._data[k] = self._verify_setter(k, v)
 
     def __delattr__(self, k):
 
-        if k.startswith('__'):
+        if k.startswith('_'):
             return object.__delattr__(self, k)
 
         del self._data[k]

From 1fc48f0cef2fa00764f13bedf7a562c2df7f633f Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Wed, 1 May 2024 18:26:30 +0200
Subject: [PATCH 14/17] simplified

---
 src/radical/utils/json_io.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/radical/utils/json_io.py b/src/radical/utils/json_io.py
index c2f768f73..3fcb5c7fd 100644
--- a/src/radical/utils/json_io.py
+++ b/src/radical/utils/json_io.py
@@ -89,16 +89,11 @@ def parse_json(json_str, filter_comments=True):
     are stripped from json before parsing
     '''
 
-    if not filter_comments:
-        return json.loads(json_str)
+    if filter_comments:
+        json_str = '\n'.join([re.sub(r'^\s*#.*$', '', line)
+                         for line in json_str.split('\n')])
 
-    else:
-        content = ''
-        for line in json_str.split('\n'):
-            content += re.sub(r'^\s*#.*$', '', line)
-            content += '\n'
-
-        return from_json(content)
+    return from_json(json_str)
 
 
 # ------------------------------------------------------------------------------

From 2445ad6f70bf2de11ec0d471de758105db261f2c Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Wed, 1 May 2024 18:26:56 +0200
Subject: [PATCH 15/17] simplified

---
 src/radical/utils/json_io.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/radical/utils/json_io.py b/src/radical/utils/json_io.py
index 3fcb5c7fd..1b954757e 100644
--- a/src/radical/utils/json_io.py
+++ b/src/radical/utils/json_io.py
@@ -6,7 +6,6 @@
 
 
 import re
-import json
 
 from .serialize import to_json, from_json
 from .misc      import as_string, ru_open

From b3c62e3f0f823633e0983c19cee0dd8caedc8632 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Thu, 2 May 2024 11:36:53 +0200
Subject: [PATCH 16/17] revert key filtering change

---
 src/radical/utils/typeddict.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 4bf38b220..49e4ff119 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -280,7 +280,7 @@ def __getattr__(self, k):
                 self.__dict__['_data'] = dict()
             return self.__dict__['_data']
 
-        if k.startswith('_'):
+        if k.startswith('__'):
             return object.__getattribute__(self, k)
 
         data   = self._data
@@ -292,14 +292,14 @@ def __getattr__(self, k):
 
     def __setattr__(self, k, v):
 
-        if k.startswith('_'):
+        if k.startswith('__'):
             return object.__setattr__(self, k, v)
 
         self._data[k] = self._verify_setter(k, v)
 
     def __delattr__(self, k):
 
-        if k.startswith('_'):
+        if k.startswith('__'):
             return object.__delattr__(self, k)
 
         del self._data[k]

From 38b82cdcd858c71e2f8ae91270d9312cdeda5b48 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Wed, 8 May 2024 22:37:10 +0200
Subject: [PATCH 17/17] response to comments

---
 src/radical/utils/serialize.py |  5 +++--
 src/radical/utils/typeddict.py | 11 ++++++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/radical/utils/serialize.py b/src/radical/utils/serialize.py
index 61c8072fe..dbace7957 100644
--- a/src/radical/utils/serialize.py
+++ b/src/radical/utils/serialize.py
@@ -2,8 +2,7 @@
 import json
 import msgpack
 
-from .typeddict import as_dict
-
+from .typeddict import as_dict, TypedDict
 
 # ------------------------------------------------------------------------------
 #
@@ -36,6 +35,8 @@ def register_serializable(cls, encode=None, decode=None):
 
     _ctypes[cls.__name__] = _CType(cls, encode, decode)
 
+register_serializable(TypedDict)
+
 
 # ------------------------------------------------------------------------------
 #
diff --git a/src/radical/utils/typeddict.py b/src/radical/utils/typeddict.py
index 49e4ff119..aeaaa6e97 100644
--- a/src/radical/utils/typeddict.py
+++ b/src/radical/utils/typeddict.py
@@ -98,7 +98,16 @@ def __new__(mcs, name, bases, namespace):
             elif k not in namespace:
                 namespace[k] = v
 
-        return super().__new__(mcs, name, bases, namespace)
+        _new_cls = super().__new__(mcs, name, bases, namespace)
+
+        if _new_cls.__base__ is not dict:
+
+            # register sub-classes
+            from .serialize import register_serializable
+            register_serializable(_new_cls)
+
+        return _new_cls
+
 
 
 # ------------------------------------------------------------------------------