diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a7bd6619..b244af68 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3380,7 +3380,7 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, logger.debug(f"Kever state: Escrowed partially delegated event=\n" f"{serder.ked}\n.") - return self.db.pdes.add(keys=snKey(serder.preb, serder.sn), val=serder.saidb) + return self.db.pdes.addOn(keys=serder.pre, on=serder.sn, val=serder.said) def state(self): @@ -5976,7 +5976,7 @@ def processEscrowPartialDels(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow # removes one event escrow at key val - self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) @@ -5988,7 +5988,7 @@ def processEscrowPartialDels(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. # removes one event escrow at key val - self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 5787c5ac..667edb2b 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -662,6 +662,65 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes + + # used in OnSuberBase + def putOnVal(self, db, key, on=0, val=b'', *, sep=b'.'): + """Write serialized bytes val to location at onkey consisting of + key + sep + serialized on in db. + Does not overwrite. + + Returns: + result (bool): True if successful write i.e onkey not already in db + False otherwise + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which write + val (bytes): to be written at onkey + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.put(onkey, val, overwrite=False)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase + def setOnVal(self, db, key, on=0, val=b'', *, sep=b'.'): + """ + Write serialized bytes val to location at onkey consisting of + key + sep + serialized on in db. + Overwrites pre-existing value at onkey if any. + + Returns: + result (bool): True if successful write i.e onkey not already in db + False otherwise + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which write + val (bytes): to be written at onkey + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.put(onkey, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase def appendOnVal(self, db, key, val, *, sep=b'.'): """ @@ -721,6 +780,35 @@ def appendOnVal(self, db, key, val, *, sep=b'.'): raise ValueError(f"Failed appending {val=} at {key=}.") return on + + # used in OnSuberBase + def getOnVal(self, db, key, on=0, *, sep=b'.'): + """Gets value at onkey consisting of key + sep + serialized on in db. + + Returns: + val (bytes | memoryview): entry at onkey consisting of key + sep + + serialized on in db. + None if no entry at key + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to retrieve + sep (bytes): separator character for split + + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return(txn.get(onkey)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase def delOnVal(self, db, key, on=0, *, sep=b'.'): """ @@ -1713,6 +1801,34 @@ def getTopIoDupItemIter(self, db, top=b''): # this is so we do the proem add and strip here not in some higher level class # like suber + def addOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): + """ + Add val bytes as dup at onkey consisting of key + sep + serialized on in db. + Adds to existing values at key if any + Returns True if written else False if dup val already exists + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if duplicate val added at onkey idempotent + False if duplicate val preexists at onkey + + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to add at onkey as dup + sep (bytes): separator character for split + """ + onkey = onKey(key, on, sep=sep) + return (self.addIoDupVal(db, key=onkey, val=val)) + + # used in OnIoDupSuber def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): """ @@ -1742,10 +1858,66 @@ def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + def delOnIoDupVals(self, db, key, on=0, sep=b'.'): + """Deletes all dup iovals at onkey consisting of key + sep + serialized + on in db. + + Assumes DB opened with dupsort=True + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if onkey present so all dups at onkey deleted + False if onkey not present + + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + sep (bytes): separator character for split + """ + onkey = onKey(key, on, sep=sep) + return (self.delIoDupVals(db, key=onkey)) + + + def delOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): + """Deletes dup ioval at key onkey consisting of key + sep + serialized + on in db. + Returns True if deleted else False if dup val not present + Assumes DB opened with dupsort=True + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if duplicate val found and deleted + False if duplicate val does not exist at onkey + + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized dup value to del at onkey + sep (bytes): separator character for split + """ + onkey = onKey(key, on, sep=sep) + return (self.delIoDupVal(db, key=onkey, val=val)) + + + # used in OnIoDupSuber def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): """ - Returns iterator of triples (key, on, val), at each key over all ordinal + Returns iterator of val at each key over all ordinal numbered keys with same key + sep + on in db. Values are sorted by onKey(key, on) where on is ordinal number int and key is prefix sans on. Values duplicates are sorted internally by hidden prefixed insertion order diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 214cf5ed..521c1788 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -436,6 +436,47 @@ def __init__(self, *pa, **kwa): super(OnSuberBase, self).__init__(*pa, **kwa) + def putOn(self, keys: str | bytes | memoryview, on: int=0, + val: str | bytes | memoryview=''): + """ + Returns + result (bool): True if onkey made from key+sep+serialized on is + not found in database so value is written + idempotently. + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + val (str | bytes | memoryview): serialization + """ + return (self.db.putOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + def pinOn(self, keys: str | bytes | memoryview, on: int=0, + val: str | bytes | memoryview=''): + """ + Returns + result (bool): True if value is written or overwritten at onkey + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + val (str | bytes | memoryview): serialization + """ + return (self.db.setOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + def appendOn(self, keys: str | bytes | memoryview, val: str | bytes | memoryview): """ @@ -446,7 +487,6 @@ def appendOn(self, keys: str | bytes | memoryview, keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form key val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(key,on) to form key. """ return (self.db.appendOnVal(db=self.sdb, key=self._tokey(keys), @@ -454,6 +494,25 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def getOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + val (str): serialization at onkey if any + None if no entry at onkey + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + val = self.db.getOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode()) + return (self._des(val) if val is not None else None) + + + def remOn(self, keys: str | bytes | memoryview, on: int=0): """ Returns @@ -2018,7 +2077,7 @@ def getLast(self, keys: str | bytes | memoryview | Iterable): def rem(self, keys: str | bytes | memoryview | Iterable, - val: str | bytes | memoryview = b''): + val: str | bytes | memoryview = ''): """ Removes entry at key made from keys and dup val that matches val if any, notwithstanding hidden ordinal proem. Otherwise deletes all dup values @@ -2141,6 +2200,32 @@ def __init__(self, *pa, **kwa): super(OnIoDupSuber, self).__init__(*pa, **kwa) + def addOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, + val: str | bytes | memoryview = ''): + """ + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(pre,on) to form onkey. + val (str | bytes | memoryview): serialization + + Returns: + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. + + """ + return (self.db.addOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + + def appendOn(self, keys: str | bytes | memoryview, val: str | bytes | memoryview): """ @@ -2151,7 +2236,6 @@ def appendOn(self, keys: str | bytes | memoryview, keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form key val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(pre,on) to form key. """ return (self.db.appendOnIoDupVal(db=self.sdb, key=self._tokey(keys), @@ -2159,6 +2243,62 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def getOn(self, keys: str | bytes | memoryview | Iterable, on: int = 0): + """ + Gets dup vals list at key made from keys + + Parameters: + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + on (int): ordinal number used with onKey(pre,on) to form key. + + Returns: + vals (list): each item in list is str + empty list if no entry at keys + + """ + return [self._des(val) for val in + self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())] + + + def remOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, + val: str | bytes | memoryview = ''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + + on (int): ordinal number used with onKey(pre,on) to form key. + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key + + Returns: + result (bool): True if onkey with dup val exists so rem successful. + False otherwise + + """ + if val: + return self.db.delOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode()) + else: + return self.db.delOnIoDupVals(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode()) + + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 7808bdc1..b6d9b4df 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -527,7 +527,7 @@ def test_missing_delegator_escrow(): psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) assert not bobPre in watKvy.kevers assert not delPre in watKvy.kevers - escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 1 assert escrows[0] == delSrdr.said # escrow entry for event @@ -538,14 +538,14 @@ def test_missing_delegator_escrow(): assert watBobK.sn == 0 watKvy.processEscrows() assert not delPre in watKvy.kevers - escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 1 assert escrows[0] == delSrdr.said # escrow entry for event # Now apply Bob's ixn to wat's kvy and process escrow psr.parse(ims=bytearray(bobIxnMsg1), kvy=watKvy, local=False) watKvy.processEscrows() - escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 0 assert watBobK.sn == 1 diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 9c2348c5..309a366d 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -311,7 +311,7 @@ def test_lmdber(): items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] assert items == [(b'b.1', b'woo')] - # test OrdVal OrdItem ordinal numbered event sub db + # test Ordinal Numbered ON keyed value methods db = dber.env.open_db(key=b'seen.') preA = b'BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' @@ -346,8 +346,18 @@ def test_lmdber(): assert dber.putVal(db, keyA0, val=digA) == False assert dber.setVal(db, keyA0, val=digA) == True assert dber.getVal(db, keyA0) == digA + assert dber.getOnVal(db, preA, 0) == digA assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None + assert dber.getOnVal(db, preA, 0) == None + + assert dber.putOnVal(db, preA, 0, val=digA) == True + assert dber.getOnVal(db, preA, 0) == digA + assert dber.putOnVal(db, preA, 0, val=digA) == False + assert dber.setOnVal(db, preA, 0, val=digA) == True + assert dber.getOnVal(db, preA, 0) == digA + assert dber.delOnVal(db, preA, 0) == True + assert dber.getOnVal(db, preA, 0) == None # test appendOnValPre # empty database @@ -981,7 +991,24 @@ def test_lmdber(): (b'Z', 1, b'l'), (b'Z', 0, b'k')] + key = b'Y' + assert dber.addOnIoDupVal(ldb, key, on=0, val=b'r') + assert dber.addOnIoDupVal(ldb, key, on=0, val=b's') + assert dber.addOnIoDupVal(ldb, key, on=1, val=b't') + assert dber.addOnIoDupVal(ldb, key, on=1, val=b'u') + + assert dber.cntOnVals(ldb, key) == 4 + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Y', 0, b'r'), + (b'Y', 0, b's'), + (b'Y', 1, b't'), + (b'Y', 1, b'u')] + assert dber.delOnIoDupVal(ldb, key, on=0, val=b's') + assert dber.delOnIoDupVals(ldb, key, on=1) + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Y', 0, b'r')] # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 6b97d3c5..58c72ccd 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -351,6 +351,14 @@ def test_on_suber(): assert items == [(('a',), 0, 'Blue dog'), (('a',), 2, 'Red apple')] + assert onsuber.putOn(keys='d', on=0, val='moon') + assert onsuber.getOn(keys='d', on=0) == 'moon' + assert not onsuber.putOn(keys='d', on=0, val='moon') + assert onsuber.pinOn(keys='d', on=0, val='sun') + assert onsuber.getOn(keys='d', on=0) == 'sun' + assert onsuber.remOn(keys='d', on=0) + + assert not os.path.exists(db.path) assert not db.opened @@ -763,6 +771,33 @@ def test_on_iodup_suber(): y = "Red apple" z = "White snow" + # test addOn remOn + assert onsuber.addOn(keys="z", on=0, val=w) + assert onsuber.getOn(keys="z", on=0) == [w] + assert onsuber.addOn(keys="z", on=0, val=x) + assert onsuber.getOn(keys="z", on=0) == [w, x] + assert onsuber.addOn(keys="z", on=1, val=y) + assert onsuber.getOn(keys="z", on=1) == [y] + assert onsuber.addOn(keys="z", on=1, val=z) + assert onsuber.getOn(keys="z", on=1) == [y, z] + + assert onsuber.cntOn(keys=("z",)) == 4 + + items = [item for item in onsuber.getOnItemIter(keys='z')] + assert items == [(('z',), 0, 'Blue dog'), + (('z',), 0, 'Green tree'), + (('z',), 1, 'Red apple'), + (('z',), 1, 'White snow')] + + assert onsuber.remOn(keys='z', on=0, val=w) + assert onsuber.remOn(keys='z', on=1) + items = [item for item in onsuber.getOnItemIter(keys='z')] + assert items == [(('z',), 0, 'Green tree')] + assert onsuber.remOn(keys='z', on=0, val=x) + + assert onsuber.cntOn(keys=("z",)) == 0 + + # test append assert 0 == onsuber.appendOn(keys=("a",), val=w) assert 1 == onsuber.appendOn(keys=("a",), val=x) @@ -1004,6 +1039,8 @@ def test_on_iodup_suber(): ] + + assert not os.path.exists(db.path) assert not db.opened