add lockable interface and object registry util

Makefile

@@ -34,6 +34,7 @@ viz:
 	gource -s 0.1 -i 0 --title radical.utils --max-files 99999 --max-file-lag -1 --user-friction 0.3 --user-scale 0.5 --camera-mode overview --highlight-users --hide progress,filenames -r 25 -viewport 1024x1024
 
 clean:
+	-rm -f pylint.out
 	-rm -rf build/ radical.utils.egg-info/ temp/ MANIFEST dist/ radical.utils.egg-info setup.cfg
 	make -C docs clean
 	find . -name \*.pyc -exec rm -f {} \;

radical/utils/lockable.py

@@ -0,0 +1,97 @@
+import threading 
+
+def Lockable (cls):
+    """ 
+    This class decorator will add lock/unlock methods to the thusly decorated
+    classes, which will be enacted via an also added `threading.RLock` member
+    (`self._rlock`)::
+
+        @Lockable
+        class A (object) :
+        
+            def call (self) :
+                print 'locked: %s' % self.locked
+        
+    The class instance can then be used like this::
+
+        a = A    ()
+        a.call   ()
+        a.lock   ()
+        a.call   ()
+        a.lock   ()
+        a.call   ()
+        a.unlock ()
+        a.call   ()
+        with a :
+          a.call ()
+        a.call   ()
+        a.unlock ()
+        a.call   ()
+    
+    which will result in::
+
+        locked: 0
+        locked: 1
+        locked: 2
+        locked: 1
+        locked: 2
+        locked: 1
+        locked: 0
+    """
+
+    if  hasattr (cls, '__enter__') :
+        raise RuntimeError ("Cannot make '%s' lockable -- has __enter__" % cls)
+
+    if  hasattr (cls, '__exit__') :
+        raise RuntimeError ("Cannot make '%s' lockable -- has __exit__" % cls)
+
+    if  hasattr (cls, '_rlock') :
+        raise RuntimeError ("Cannot make '%s' lockable -- has _rlock" % cls)
+
+    if  hasattr(cls, 'locked') :
+        raise RuntimeError ("Cannot make '%s' lockable -- has locked" % cls)
+
+    if  hasattr (cls, 'lock') :
+        raise RuntimeError ("Cannot make '%s' lockable -- has lock()" % cls)
+
+    if  hasattr (cls, 'unlock') :
+        raise RuntimeError ("Cannot make '%s' lockable -- has unlock()" % cls)
+
+
+    def locker   (self)        : self._rlock.acquire (); self.locked += 1
+    def unlocker (self, *args) : self._rlock.release (); self.locked -= 1
+
+    cls._rlock    = threading.RLock ()
+    cls.locked    = 0
+    cls.lock      = locker
+    cls.unlock    = unlocker
+    cls.__enter__ = locker
+    cls.__exit__  = unlocker
+
+    return cls
+
+
+
+# @Lockable
+# class A (object) :
+# 
+#     def call (self) :
+#         print 'locked: %s' % self.locked
+# 
+# a = A()
+# a.call ()
+# a.lock ()
+# a.call ()
+# a.lock ()
+# a.call ()
+# a.unlock ()
+# a.call ()
+# with a :
+#   a.call ()
+# a.call ()
+# a.unlock ()
+# a.call ()
+
+# ------------------------------------------------------------------------------
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
+

radical/utils/registry.py

@@ -0,0 +1,281 @@
+
+import time
+import threading
+import contextlib
+import radical.utils as ru
+
+
+# ------------------------------------------------------------------------------
+#
+TIMEOUT   = 0.1 # sleep between lock checks
+READONLY  = 'ReadOnly'
+READWRITE = 'ReadWrite'
+# ------------------------------------------------------------------------------
+#
+Registry = _Registry ()
+
+
+# ------------------------------------------------------------------------------
+#
+class _Registry (object) :
+    """
+    This singleton registry class maintains a set of object instances, by some
+    entity ID.  Consumers can acquire and release those instances, for
+    `READONLY` or `READWRITE` activities.  An unlimited number of READONLY
+    activities can be active at any time, on any given instance -- but if any
+    `READWRITE` acquisition will block until no other instances are leased
+    anymore, and any `READONLY` acquisition will block until an eventual release
+    of a `READWRITE` lease::
+
+        my_obj = MyClass (eid='my_id')
+        print my_obj.id   # prints my_id
+        ...
+
+        import raducal.utils as  ru
+        ro_instance_1 = ru.Registry.acquire ('my_id', ru.READONLY)
+        ro_instance_2 = ru.Registry.acquire ('my_id', ru.READONLY)
+        ...
+
+        rw_instance_1 = ru.Registry.acquire ('my_id', ru.READWRITE)
+
+        # the last call would block until the following two calls have been
+        # issued:
+        ru.Registry.release ('my_id')
+        ru.Registry.release ('my_id')
+
+    The correct semantic functionality of this class depends on the careful
+    release of unused instances.  Using a `with` statement is encouraged, and
+    a `lease` context manager is provided for that purpose::
+
+        with ru.Registry.lease (eid, ru.READONLY) as ro_instance :
+            ro_instance.do_something ()
+            ro_instance.do_nothing   ()
+            ro_instance.do_something ()
+
+        with ru.Registry.lease (eid, ru.READWRITE) as rw_instance :
+            rw_instance.change_something
+
+    The registry will thus ensure that a consumer will always see instances
+    which are not changed by a third party over the scope of a lease.
+
+    *Requirements*
+
+    Registered object instances must fulfill two requirements:
+    
+    * they must be `lockable`, i.e. during `aquire` we can call `entity.lock()`
+      on them to activate a `threading.RLock`, and during `release()` we can
+      call `entity.unlock()` to deactivate it.  Instances will thus guaranteed
+      to be locked during lease.
+
+    * they must be `identifiable`, i.e. they must have an `id` property.
+      Alternatively, an entity ID can be passed as optional parameter during
+      registration -- all followup methods will require that eid.  
+
+
+    *Note* (for Troy as consumer of this registry, but applicable in general):
+
+    It is in general not a favourable design to have large, all-visible
+    registries in a multi-component software stack, as ownership of state
+    transitions can easily become blurry, and as a registry can also become
+    a performance bottleneck on frequent queries -- so why are we doing this?
+
+    First of all, state transitions in Troy *are* blurry to some extent, as Troy
+    aims to support a variety of state diagram transitions, and thus the order
+    of transitions is not pre-defined (first derive overlay then schedule CUs,
+    or vice versa?).  Also, the ownership of state transitions for a workload is
+    not easily scoped (the :class:`Planner` will move a workload from `DESCRIBED` 
+    to `PLANNED`, the :class:`WorkloadManager` will move it from `PLANNED` to
+    `TRANSLATED`, etc.  And, finally, we want to allow for re-scheduling,
+    re-translation, re-planning etc, which would require us to pass control of
+    a workload back and forth between different modules.  Finally, this seems to
+    be useful for inspection and introspection of Troy activities related to
+    specific workload instances.
+
+    In that context, a registry seems the (much) lesser of two devils: The
+    registry class will allow to register workload and overlay instances, and to
+    acquire/release control over them.  The module which acquires control needs
+    to ascertain that the respective workload and overlay instances are in
+    a usable state for that module -- the registry is neither interpreting nor
+    enforcing any state model on the managed instances -- that is up to the
+    leasing module.  Neither will the registry perform garbage collection --
+    active unregistration will remove instances from the registry, but not
+    enforce a deletion.
+
+    This is a singleton class.  We assume that Workload *and* Overlay IDs are
+    unique.
+    """
+    __metaclass__ = ru.Singleton
+
+
+    # --------------------------------------------------------------------------
+    #
+    def __init__ (self) :
+        """
+        Create a new Registry instance.  
+        """
+
+        # make this instance lockable
+        self.lock = threading.RLock ()
+
+        self._registry = dict()
+        self._session  = None  # this will be set by Troy.submit_workload()
+
+
+    # ------------------------------------------------------------------------------
+    #
+    @contextlib.contextmanager
+    def lease (self, oid, mode=READWRITE) :
+        try :
+            yield self.acquire (oid, mode)
+        finally :
+            self.release (oid)
+
+    # --------------------------------------------------------------------------
+    #
+    def register (self, entity, eid=None) :
+        """
+        register a new object instance.
+        """
+
+        # lock manager before checking/manipulating the registry
+        with self.lock :
+
+            # check if instance is lockable
+            if  not hasattr (entity, '__enter__') or \
+                not hasattr (entity, '__exit__' ) or \
+                not hasattr (entity, '__lock'   ) or \
+                not hasattr (entity, '__unlock' ) or \
+                not hasattr (entity, '_rlock'   ) :
+                raise TypeError ("Registry only manages lockables")
+
+            # check if instance is identifiable
+            if  not eid :
+                if  not hasattr (entity, 'id') :
+                    raise TypeError ("Registry only manages identifiables")
+                eid = entity.id
+
+            if  eid in self._registry :
+                raise ValueError ("'%s' is already registered" % eid)
+
+            print 'register %s' % eid
+
+            self._registry[eid] = {}
+            self._registry[eid]['ro_leased'] = 0  # not leased
+            self._registry[eid]['rw_leased'] = 0  # not leased
+            self._registry[eid]['entity']    = entity
+
+
+    # --------------------------------------------------------------------------
+    #
+    def acquire (self, eid, mode) :
+        """
+        temporarily relinquish control over the referenced identity to the
+        caller.
+        """
+
+        # sanity check
+        if  not eid in self._registry :
+            KeyError ("'%s' is not registered" % eid)
+
+        # wait for the entity to be fee for the expected usage
+        while True :
+
+            # lock manager before checking/manipulating the registry
+            with  self.lock :
+
+                if  mode == READONLY :
+                    # make sure we have no active write lease
+
+                    if  self._registry[eid]['rw_leases'] :
+                        # need to wait
+                        time.sleep (TIMEOUT)
+                        continue
+
+                    else :
+                        # free for READONLY use
+                        self._registry[eid]['ro_leases'] += 1
+                        break
+
+                if  mode == READWRITE :
+                    # make sure we have no active read or write lease
+
+                    if  self._registry[eid]['rw_leases'] or \
+                        self._registry[eid]['ro_leases'] :
+                        # need to wait
+                        time.sleep (TIMEOUT)
+                        continue
+
+                    else :
+                        # free for READWRITE use
+                        self._registry[eid]['rw_leases'] += 1
+                        break
+
+            # acquire entity lock
+            self._registry[eid]['entity'].lock ()
+
+            # all is well...
+            return self._registry[eid]['entity']
+
+
+    # --------------------------------------------------------------------------
+    #
+    def release (self, eid) :
+        """
+        relinquish the control over the referenced entity
+        """
+
+        # sanity check
+        if  not eid in self._registry :
+            raise KeyError ("'%s' is not registered" % eid)
+
+        # lock manager before checking/manipulating the registry
+        with  self.lock :
+
+            if  not self._registry[eid]['ro_leases'] and \
+                not self._registry[eid]['rw_leases'] :
+                raise ValueError ("'%s' was not acquired" % eid)
+
+            # release entity lease
+            if  self._registry[eid]['ro_leases'] :
+                self._registry[eid]['ro_leases'] -= 1
+            elif self._registry[eid]['rw_leases'] :
+                self._registry[eid]['rw_leases'] -= 1
+
+            # release entity lock
+            self._registry[eid]['entity'].lock.release ()
+
+            # all is well...
+
+
+    # --------------------------------------------------------------------------
+    #
+    def unregister (self, eid) :
+        """
+        remove the reference entity from the registry, but do not explicitly
+        call the entity's destructor.  This will unlock the entity.
+        """
+
+        # sanity check
+        if  not eid in self._registry :
+            raise KeyError ("'%s' is not registered" % eid)
+
+        # lock manager before checking/manipulating the registry
+        with  self.lock :
+
+            # unlock entity
+            while self._registry[eid]['ro_leases'] :
+                self._registry[eid]['entity'].unlock ()
+                self._registry[eid]['ro_leases'] -= 1
+
+            while self._registry[eid]['rw_leases'] :
+                self._registry[eid]['entity'].unlock ()
+                self._registry[eid]['rw_leases'] -= 1
+
+
+            # remove entity from registry, w/o a trace...
+            del self._registry[eid]
+
+
+# ------------------------------------------------------------------------------
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
+