diff --git a/Makefile b/Makefile index d44a1239f..29f8e29cb 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ viz: gource -s 0.1 -i 0 --title radical.utils --max-files 99999 --max-file-lag -1 --user-friction 0.3 --user-scale 0.5 --camera-mode overview --highlight-users --hide progress,filenames -r 25 -viewport 1024x1024 clean: + -rm -f pylint.out -rm -rf build/ radical.utils.egg-info/ temp/ MANIFEST dist/ radical.utils.egg-info setup.cfg make -C docs clean find . -name \*.pyc -exec rm -f {} \; diff --git a/radical/utils/lockable.py b/radical/utils/lockable.py new file mode 100644 index 000000000..28a3f1b9c --- /dev/null +++ b/radical/utils/lockable.py @@ -0,0 +1,97 @@ +import threading + +def Lockable (cls): + """ + This class decorator will add lock/unlock methods to the thusly decorated + classes, which will be enacted via an also added `threading.RLock` member + (`self._rlock`):: + + @Lockable + class A (object) : + + def call (self) : + print 'locked: %s' % self.locked + + The class instance can then be used like this:: + + a = A () + a.call () + a.lock () + a.call () + a.lock () + a.call () + a.unlock () + a.call () + with a : + a.call () + a.call () + a.unlock () + a.call () + + which will result in:: + + locked: 0 + locked: 1 + locked: 2 + locked: 1 + locked: 2 + locked: 1 + locked: 0 + """ + + if hasattr (cls, '__enter__') : + raise RuntimeError ("Cannot make '%s' lockable -- has __enter__" % cls) + + if hasattr (cls, '__exit__') : + raise RuntimeError ("Cannot make '%s' lockable -- has __exit__" % cls) + + if hasattr (cls, '_rlock') : + raise RuntimeError ("Cannot make '%s' lockable -- has _rlock" % cls) + + if hasattr(cls, 'locked') : + raise RuntimeError ("Cannot make '%s' lockable -- has locked" % cls) + + if hasattr (cls, 'lock') : + raise RuntimeError ("Cannot make '%s' lockable -- has lock()" % cls) + + if hasattr (cls, 'unlock') : + raise RuntimeError ("Cannot make '%s' lockable -- has unlock()" % cls) + + + def locker (self) : self._rlock.acquire (); self.locked += 1 + def unlocker (self, *args) : self._rlock.release (); self.locked -= 1 + + cls._rlock = threading.RLock () + cls.locked = 0 + cls.lock = locker + cls.unlock = unlocker + cls.__enter__ = locker + cls.__exit__ = unlocker + + return cls + + + +# @Lockable +# class A (object) : +# +# def call (self) : +# print 'locked: %s' % self.locked +# +# a = A() +# a.call () +# a.lock () +# a.call () +# a.lock () +# a.call () +# a.unlock () +# a.call () +# with a : +# a.call () +# a.call () +# a.unlock () +# a.call () + +# ------------------------------------------------------------------------------ +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 + diff --git a/radical/utils/registry.py b/radical/utils/registry.py new file mode 100644 index 000000000..7c8b9b217 --- /dev/null +++ b/radical/utils/registry.py @@ -0,0 +1,281 @@ + +import time +import threading +import contextlib +import radical.utils as ru + + +# ------------------------------------------------------------------------------ +# +TIMEOUT = 0.1 # sleep between lock checks +READONLY = 'ReadOnly' +READWRITE = 'ReadWrite' +# ------------------------------------------------------------------------------ +# +Registry = _Registry () + + +# ------------------------------------------------------------------------------ +# +class _Registry (object) : + """ + This singleton registry class maintains a set of object instances, by some + entity ID. Consumers can acquire and release those instances, for + `READONLY` or `READWRITE` activities. An unlimited number of READONLY + activities can be active at any time, on any given instance -- but if any + `READWRITE` acquisition will block until no other instances are leased + anymore, and any `READONLY` acquisition will block until an eventual release + of a `READWRITE` lease:: + + my_obj = MyClass (eid='my_id') + print my_obj.id # prints my_id + ... + + import raducal.utils as ru + ro_instance_1 = ru.Registry.acquire ('my_id', ru.READONLY) + ro_instance_2 = ru.Registry.acquire ('my_id', ru.READONLY) + ... + + rw_instance_1 = ru.Registry.acquire ('my_id', ru.READWRITE) + + # the last call would block until the following two calls have been + # issued: + ru.Registry.release ('my_id') + ru.Registry.release ('my_id') + + The correct semantic functionality of this class depends on the careful + release of unused instances. Using a `with` statement is encouraged, and + a `lease` context manager is provided for that purpose:: + + with ru.Registry.lease (eid, ru.READONLY) as ro_instance : + ro_instance.do_something () + ro_instance.do_nothing () + ro_instance.do_something () + + with ru.Registry.lease (eid, ru.READWRITE) as rw_instance : + rw_instance.change_something + + The registry will thus ensure that a consumer will always see instances + which are not changed by a third party over the scope of a lease. + + *Requirements* + + Registered object instances must fulfill two requirements: + + * they must be `lockable`, i.e. during `aquire` we can call `entity.lock()` + on them to activate a `threading.RLock`, and during `release()` we can + call `entity.unlock()` to deactivate it. Instances will thus guaranteed + to be locked during lease. + + * they must be `identifiable`, i.e. they must have an `id` property. + Alternatively, an entity ID can be passed as optional parameter during + registration -- all followup methods will require that eid. + + + *Note* (for Troy as consumer of this registry, but applicable in general): + + It is in general not a favourable design to have large, all-visible + registries in a multi-component software stack, as ownership of state + transitions can easily become blurry, and as a registry can also become + a performance bottleneck on frequent queries -- so why are we doing this? + + First of all, state transitions in Troy *are* blurry to some extent, as Troy + aims to support a variety of state diagram transitions, and thus the order + of transitions is not pre-defined (first derive overlay then schedule CUs, + or vice versa?). Also, the ownership of state transitions for a workload is + not easily scoped (the :class:`Planner` will move a workload from `DESCRIBED` + to `PLANNED`, the :class:`WorkloadManager` will move it from `PLANNED` to + `TRANSLATED`, etc. And, finally, we want to allow for re-scheduling, + re-translation, re-planning etc, which would require us to pass control of + a workload back and forth between different modules. Finally, this seems to + be useful for inspection and introspection of Troy activities related to + specific workload instances. + + In that context, a registry seems the (much) lesser of two devils: The + registry class will allow to register workload and overlay instances, and to + acquire/release control over them. The module which acquires control needs + to ascertain that the respective workload and overlay instances are in + a usable state for that module -- the registry is neither interpreting nor + enforcing any state model on the managed instances -- that is up to the + leasing module. Neither will the registry perform garbage collection -- + active unregistration will remove instances from the registry, but not + enforce a deletion. + + This is a singleton class. We assume that Workload *and* Overlay IDs are + unique. + """ + __metaclass__ = ru.Singleton + + + # -------------------------------------------------------------------------- + # + def __init__ (self) : + """ + Create a new Registry instance. + """ + + # make this instance lockable + self.lock = threading.RLock () + + self._registry = dict() + self._session = None # this will be set by Troy.submit_workload() + + + # ------------------------------------------------------------------------------ + # + @contextlib.contextmanager + def lease (self, oid, mode=READWRITE) : + try : + yield self.acquire (oid, mode) + finally : + self.release (oid) + + # -------------------------------------------------------------------------- + # + def register (self, entity, eid=None) : + """ + register a new object instance. + """ + + # lock manager before checking/manipulating the registry + with self.lock : + + # check if instance is lockable + if not hasattr (entity, '__enter__') or \ + not hasattr (entity, '__exit__' ) or \ + not hasattr (entity, '__lock' ) or \ + not hasattr (entity, '__unlock' ) or \ + not hasattr (entity, '_rlock' ) : + raise TypeError ("Registry only manages lockables") + + # check if instance is identifiable + if not eid : + if not hasattr (entity, 'id') : + raise TypeError ("Registry only manages identifiables") + eid = entity.id + + if eid in self._registry : + raise ValueError ("'%s' is already registered" % eid) + + print 'register %s' % eid + + self._registry[eid] = {} + self._registry[eid]['ro_leased'] = 0 # not leased + self._registry[eid]['rw_leased'] = 0 # not leased + self._registry[eid]['entity'] = entity + + + # -------------------------------------------------------------------------- + # + def acquire (self, eid, mode) : + """ + temporarily relinquish control over the referenced identity to the + caller. + """ + + # sanity check + if not eid in self._registry : + KeyError ("'%s' is not registered" % eid) + + # wait for the entity to be fee for the expected usage + while True : + + # lock manager before checking/manipulating the registry + with self.lock : + + if mode == READONLY : + # make sure we have no active write lease + + if self._registry[eid]['rw_leases'] : + # need to wait + time.sleep (TIMEOUT) + continue + + else : + # free for READONLY use + self._registry[eid]['ro_leases'] += 1 + break + + if mode == READWRITE : + # make sure we have no active read or write lease + + if self._registry[eid]['rw_leases'] or \ + self._registry[eid]['ro_leases'] : + # need to wait + time.sleep (TIMEOUT) + continue + + else : + # free for READWRITE use + self._registry[eid]['rw_leases'] += 1 + break + + # acquire entity lock + self._registry[eid]['entity'].lock () + + # all is well... + return self._registry[eid]['entity'] + + + # -------------------------------------------------------------------------- + # + def release (self, eid) : + """ + relinquish the control over the referenced entity + """ + + # sanity check + if not eid in self._registry : + raise KeyError ("'%s' is not registered" % eid) + + # lock manager before checking/manipulating the registry + with self.lock : + + if not self._registry[eid]['ro_leases'] and \ + not self._registry[eid]['rw_leases'] : + raise ValueError ("'%s' was not acquired" % eid) + + # release entity lease + if self._registry[eid]['ro_leases'] : + self._registry[eid]['ro_leases'] -= 1 + elif self._registry[eid]['rw_leases'] : + self._registry[eid]['rw_leases'] -= 1 + + # release entity lock + self._registry[eid]['entity'].lock.release () + + # all is well... + + + # -------------------------------------------------------------------------- + # + def unregister (self, eid) : + """ + remove the reference entity from the registry, but do not explicitly + call the entity's destructor. This will unlock the entity. + """ + + # sanity check + if not eid in self._registry : + raise KeyError ("'%s' is not registered" % eid) + + # lock manager before checking/manipulating the registry + with self.lock : + + # unlock entity + while self._registry[eid]['ro_leases'] : + self._registry[eid]['entity'].unlock () + self._registry[eid]['ro_leases'] -= 1 + + while self._registry[eid]['rw_leases'] : + self._registry[eid]['entity'].unlock () + self._registry[eid]['rw_leases'] -= 1 + + + # remove entity from registry, w/o a trace... + del self._registry[eid] + + +# ------------------------------------------------------------------------------ +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +